2024-12-04 14:11:28,262 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc 2024-12-04 14:11:28,274 main DEBUG Took 0.010566 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 14:11:28,275 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 14:11:28,275 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 14:11:28,276 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 14:11:28,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,283 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 14:11:28,294 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,296 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,297 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,298 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,299 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,300 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,300 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,301 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,302 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,302 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,303 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,304 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:11:28,304 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,305 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 14:11:28,306 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:11:28,307 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 14:11:28,309 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 14:11:28,309 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 14:11:28,310 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 14:11:28,310 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 14:11:28,318 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 14:11:28,321 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 14:11:28,323 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 14:11:28,323 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 14:11:28,323 main DEBUG createAppenders(={Console}) 2024-12-04 14:11:28,324 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc initialized 2024-12-04 14:11:28,324 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc 2024-12-04 14:11:28,324 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc OK. 2024-12-04 14:11:28,325 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 14:11:28,325 main DEBUG OutputStream closed 2024-12-04 14:11:28,325 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 14:11:28,325 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 14:11:28,326 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53ce1329 OK 2024-12-04 14:11:28,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 14:11:28,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 14:11:28,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 14:11:28,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 14:11:28,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 14:11:28,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 14:11:28,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 14:11:28,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 14:11:28,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 14:11:28,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 14:11:28,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 14:11:28,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 14:11:28,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 14:11:28,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 14:11:28,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 14:11:28,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 14:11:28,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 14:11:28,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 14:11:28,402 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 14:11:28,403 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-12-04 14:11:28,403 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 14:11:28,404 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-12-04T14:11:28,755 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70 2024-12-04 14:11:28,759 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 14:11:28,759 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T14:11:28,772 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-12-04T14:11:28,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=286, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=7205 2024-12-04T14:11:28,818 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:11:28,821 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf, deleteOnExit=true 2024-12-04T14:11:28,822 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-04T14:11:28,823 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/test.cache.data in system properties and HBase conf 2024-12-04T14:11:28,823 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:11:28,824 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:11:28,825 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:11:28,826 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:11:28,826 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-04T14:11:28,925 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T14:11:29,027 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:11:29,032 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:11:29,032 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:11:29,033 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:11:29,033 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:11:29,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:11:29,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:11:29,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:11:29,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:11:29,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:11:29,037 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:11:29,038 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:11:29,038 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:11:29,039 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:11:29,039 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:11:29,572 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:11:29,920 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T14:11:30,002 INFO [Time-limited test {}] log.Log(170): Logging initialized @2472ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T14:11:30,105 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:11:30,190 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:11:30,213 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:11:30,214 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:11:30,215 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:11:30,230 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:11:30,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:11:30,235 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:11:30,441 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5682c4d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/java.io.tmpdir/jetty-localhost-34505-hadoop-hdfs-3_4_1-tests_jar-_-any-6103146150170019993/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:11:30,452 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:34505} 2024-12-04T14:11:30,452 INFO [Time-limited test {}] server.Server(415): Started @2923ms 2024-12-04T14:11:30,485 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:11:30,860 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:11:30,868 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:11:30,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:11:30,870 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:11:30,870 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:11:30,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:11:30,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:11:31,000 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6aad8790{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/java.io.tmpdir/jetty-localhost-44233-hadoop-hdfs-3_4_1-tests_jar-_-any-15514229907644836236/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:11:31,001 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:44233} 2024-12-04T14:11:31,001 INFO [Time-limited test {}] server.Server(415): Started @3471ms 2024-12-04T14:11:31,063 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:11:31,195 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:11:31,207 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:11:31,208 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:11:31,208 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:11:31,209 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:11:31,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:11:31,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:11:31,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@163cfad6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/java.io.tmpdir/jetty-localhost-46219-hadoop-hdfs-3_4_1-tests_jar-_-any-9497304854087895432/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:11:31,371 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:46219} 2024-12-04T14:11:31,372 INFO [Time-limited test {}] server.Server(415): Started @3842ms 2024-12-04T14:11:31,374 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:11:31,533 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data3/current/BP-802734383-172.17.0.2-1733321489678/current, will proceed with Du for space computation calculation, 2024-12-04T14:11:31,533 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data2/current/BP-802734383-172.17.0.2-1733321489678/current, will proceed with Du for space computation calculation, 2024-12-04T14:11:31,533 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data1/current/BP-802734383-172.17.0.2-1733321489678/current, will proceed with Du for space computation calculation, 2024-12-04T14:11:31,536 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data4/current/BP-802734383-172.17.0.2-1733321489678/current, will proceed with Du for space computation calculation, 2024-12-04T14:11:31,605 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:11:31,606 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:11:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbcbc130e052388fe with lease ID 0x38c2e7c263617ea8: Processing first storage report for DS-929b6647-d0dd-46dc-b58d-4c259bd3016f from datanode DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e1aa4844-70c0-46cc-ae59-f5d904479534, infoPort=44197, infoSecurePort=0, ipcPort=35689, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678) 2024-12-04T14:11:31,695 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbcbc130e052388fe with lease ID 0x38c2e7c263617ea8: from storage DS-929b6647-d0dd-46dc-b58d-4c259bd3016f node DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e1aa4844-70c0-46cc-ae59-f5d904479534, infoPort=44197, infoSecurePort=0, ipcPort=35689, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-04T14:11:31,695 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdc3b7ee1f2afc609 with lease ID 0x38c2e7c263617ea9: Processing first storage report for DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee from datanode DatanodeRegistration(127.0.0.1:41151, datanodeUuid=3180f54a-891e-48b6-866d-0ef0be60ea1e, infoPort=45005, infoSecurePort=0, ipcPort=40169, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678) 2024-12-04T14:11:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc3b7ee1f2afc609 with lease ID 0x38c2e7c263617ea9: from storage DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee node DatanodeRegistration(127.0.0.1:41151, datanodeUuid=3180f54a-891e-48b6-866d-0ef0be60ea1e, infoPort=45005, infoSecurePort=0, ipcPort=40169, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:11:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbcbc130e052388fe with lease ID 0x38c2e7c263617ea8: Processing first storage report for DS-2a19bca3-1834-40d4-8e60-dd45e1d96fce from datanode DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e1aa4844-70c0-46cc-ae59-f5d904479534, infoPort=44197, infoSecurePort=0, ipcPort=35689, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678) 2024-12-04T14:11:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbcbc130e052388fe with lease ID 0x38c2e7c263617ea8: from storage DS-2a19bca3-1834-40d4-8e60-dd45e1d96fce node DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e1aa4844-70c0-46cc-ae59-f5d904479534, infoPort=44197, infoSecurePort=0, ipcPort=35689, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:11:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdc3b7ee1f2afc609 with lease ID 0x38c2e7c263617ea9: Processing first storage report for DS-3dca2527-d33a-4295-a68a-4fb08faa26a8 from datanode DatanodeRegistration(127.0.0.1:41151, datanodeUuid=3180f54a-891e-48b6-866d-0ef0be60ea1e, infoPort=45005, infoSecurePort=0, ipcPort=40169, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678) 2024-12-04T14:11:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc3b7ee1f2afc609 with lease ID 0x38c2e7c263617ea9: from storage DS-3dca2527-d33a-4295-a68a-4fb08faa26a8 node DatanodeRegistration(127.0.0.1:41151, datanodeUuid=3180f54a-891e-48b6-866d-0ef0be60ea1e, infoPort=45005, infoSecurePort=0, ipcPort=40169, storageInfo=lv=-57;cid=testClusterID;nsid=1252296470;c=1733321489678), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:11:31,794 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70 2024-12-04T14:11:31,883 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/zookeeper_0, clientPort=63185, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:11:31,897 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=63185 2024-12-04T14:11:31,911 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:31,914 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:32,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:11:32,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:11:32,581 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78 with version=8 2024-12-04T14:11:32,581 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase-staging 2024-12-04T14:11:32,719 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T14:11:33,005 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:11:33,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:11:33,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:11:33,026 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:11:33,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:11:33,027 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:11:33,179 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:11:33,242 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T14:11:33,252 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T14:11:33,256 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:11:33,286 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 29845 (auto-detected) 2024-12-04T14:11:33,287 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T14:11:33,308 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40617 2024-12-04T14:11:33,316 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:33,319 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:33,332 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40617 connecting to ZooKeeper ensemble=127.0.0.1:63185 2024-12-04T14:11:33,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:406170x0, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:11:33,367 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40617-0x1005d5ebe780000 connected 2024-12-04T14:11:33,395 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:11:33,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:11:33,401 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:11:33,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40617 2024-12-04T14:11:33,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40617 2024-12-04T14:11:33,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40617 2024-12-04T14:11:33,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40617 2024-12-04T14:11:33,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40617 2024-12-04T14:11:33,416 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78, hbase.cluster.distributed=false 2024-12-04T14:11:33,505 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:11:33,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:11:33,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:11:33,506 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:11:33,507 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:11:33,507 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:11:33,510 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:11:33,513 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:11:33,514 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46401 2024-12-04T14:11:33,516 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:11:33,522 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:11:33,524 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:33,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:33,536 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:46401 connecting to ZooKeeper ensemble=127.0.0.1:63185 2024-12-04T14:11:33,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464010x0, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:11:33,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464010x0, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:11:33,541 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46401-0x1005d5ebe780001 connected 2024-12-04T14:11:33,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:11:33,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:11:33,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46401 2024-12-04T14:11:33,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46401 2024-12-04T14:11:33,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46401 2024-12-04T14:11:33,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46401 2024-12-04T14:11:33,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46401 2024-12-04T14:11:33,552 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:11:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:11:33,564 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:33,569 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec15ac6edd88:40617 2024-12-04T14:11:33,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:11:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:11:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:33,598 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:11:33,599 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec15ac6edd88,40617,1733321492712 from backup master directory 2024-12-04T14:11:33,599 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:11:33,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:33,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:11:33,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:11:33,603 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:11:33,603 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:33,605 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T14:11:33,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T14:11:33,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:11:33,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:11:33,692 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase.id with ID: e869baa9-0a42-4ea1-a12f-7c66de0212e6 2024-12-04T14:11:33,738 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:33,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:33,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:33,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:11:33,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:11:33,802 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:11:33,804 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:11:33,810 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:11:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:11:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:11:33,868 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store 2024-12-04T14:11:33,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:11:33,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:11:33,891 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T14:11:33,892 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:33,893 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:11:33,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:11:33,894 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:11:33,894 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:11:33,894 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:11:33,894 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:11:33,894 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:11:33,896 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/.initializing 2024-12-04T14:11:33,897 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/WALs/ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:33,914 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C40617%2C1733321492712, suffix=, logDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/WALs/ec15ac6edd88,40617,1733321492712, archiveDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/oldWALs, maxLogs=10 2024-12-04T14:11:33,925 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40617%2C1733321492712.1733321493922 2024-12-04T14:11:33,926 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag. 2024-12-04T14:11:33,926 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag. 2024-12-04T14:11:33,948 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/WALs/ec15ac6edd88,40617,1733321492712/ec15ac6edd88%2C40617%2C1733321492712.1733321493922 2024-12-04T14:11:33,957 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45005:45005),(127.0.0.1/127.0.0.1:44197:44197)] 2024-12-04T14:11:33,957 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:11:33,958 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:33,961 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:33,963 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:11:34,030 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:34,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,037 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:11:34,038 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:11:34,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:11:34,042 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:11:34,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:11:34,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:11:34,051 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,053 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,062 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:11:34,065 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:11:34,069 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:11:34,070 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812127, jitterRate=0.032673180103302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:11:34,074 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:11:34,075 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:11:34,105 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab442c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:11:34,140 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-04T14:11:34,152 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:11:34,152 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:11:34,155 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:11:34,156 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T14:11:34,161 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-12-04T14:11:34,161 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:11:34,187 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:11:34,201 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:11:34,203 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:11:34,205 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:11:34,207 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:11:34,208 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:11:34,210 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:11:34,214 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:11:34,216 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:11:34,217 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:11:34,218 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:11:34,227 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:11:34,229 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:11:34,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:11:34,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:11:34,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,233 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ec15ac6edd88,40617,1733321492712, sessionid=0x1005d5ebe780000, setting cluster-up flag (Was=false) 2024-12-04T14:11:34,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,251 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:11:34,252 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:34,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,264 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:11:34,265 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:34,345 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-04T14:11:34,351 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-04T14:11:34,354 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:11:34,360 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec15ac6edd88,40617,1733321492712 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:11:34,362 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec15ac6edd88:46401 2024-12-04T14:11:34,363 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:11:34,363 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:11:34,364 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:11:34,364 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:11:34,364 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec15ac6edd88:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:11:34,364 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1008): ClusterId : e869baa9-0a42-4ea1-a12f-7c66de0212e6 2024-12-04T14:11:34,364 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,364 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:11:34,364 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,366 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733321524366 2024-12-04T14:11:34,367 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:11:34,368 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:11:34,368 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:11:34,370 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:11:34,371 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:11:34,371 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:11:34,372 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:11:34,372 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:11:34,373 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:11:34,373 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:11:34,373 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:11:34,373 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,374 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:11:34,375 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:11:34,375 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,376 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:11:34,376 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:11:34,376 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:11:34,377 DEBUG [RS:0;ec15ac6edd88:46401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb5ac42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:11:34,379 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:11:34,379 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:11:34,379 DEBUG [RS:0;ec15ac6edd88:46401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b42877e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:11:34,382 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321494380,5,FailOnTimeoutGroup] 2024-12-04T14:11:34,383 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:11:34,383 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:11:34,383 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:11:34,384 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321494382,5,FailOnTimeoutGroup] 2024-12-04T14:11:34,384 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,384 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:11:34,385 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,386 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,386 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,40617,1733321492712 with isa=ec15ac6edd88/172.17.0.2:46401, startcode=1733321493504 2024-12-04T14:11:34,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:11:34,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:11:34,391 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-04T14:11:34,391 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78 2024-12-04T14:11:34,401 DEBUG [RS:0;ec15ac6edd88:46401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:11:34,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:11:34,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:11:34,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:34,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:11:34,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:11:34,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:34,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:11:34,433 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:11:34,433 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:34,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:11:34,436 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:11:34,436 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:34,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:34,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740 2024-12-04T14:11:34,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740 2024-12-04T14:11:34,442 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:11:34,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:11:34,450 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:11:34,451 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=698987, jitterRate=-0.11119313538074493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:11:34,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:11:34,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:11:34,455 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:11:34,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:11:34,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:11:34,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:11:34,456 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:11:34,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:11:34,459 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:11:34,459 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-04T14:11:34,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:11:34,473 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:11:34,475 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:11:34,485 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54675, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:11:34,491 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40617 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,494 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40617 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,509 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78 2024-12-04T14:11:34,509 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37417 2024-12-04T14:11:34,509 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:11:34,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:11:34,514 DEBUG [RS:0;ec15ac6edd88:46401 {}] zookeeper.ZKUtil(111): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,514 WARN [RS:0;ec15ac6edd88:46401 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:11:34,514 INFO [RS:0;ec15ac6edd88:46401 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:11:34,515 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,517 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,46401,1733321493504] 2024-12-04T14:11:34,530 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:11:34,545 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:11:34,557 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:11:34,560 INFO [RS:0;ec15ac6edd88:46401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:11:34,560 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,561 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:11:34,568 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,568 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,569 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,570 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:11:34,570 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:11:34,570 DEBUG [RS:0;ec15ac6edd88:46401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:11:34,572 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,572 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,572 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,572 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,573 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321493504-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:11:34,592 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:11:34,594 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321493504-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:34,611 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.Replication(204): ec15ac6edd88,46401,1733321493504 started 2024-12-04T14:11:34,612 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,46401,1733321493504, RpcServer on ec15ac6edd88/172.17.0.2:46401, sessionid=0x1005d5ebe780001 2024-12-04T14:11:34,612 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:11:34,612 DEBUG [RS:0;ec15ac6edd88:46401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,612 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,46401,1733321493504' 2024-12-04T14:11:34,612 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:11:34,613 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:11:34,614 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:11:34,614 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:11:34,614 DEBUG [RS:0;ec15ac6edd88:46401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,614 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,46401,1733321493504' 2024-12-04T14:11:34,614 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:11:34,615 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:11:34,616 DEBUG [RS:0;ec15ac6edd88:46401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:11:34,616 INFO [RS:0;ec15ac6edd88:46401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:11:34,616 INFO [RS:0;ec15ac6edd88:46401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:11:34,626 WARN [ec15ac6edd88:40617 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-04T14:11:34,724 INFO [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C46401%2C1733321493504, suffix=, logDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504, archiveDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs, maxLogs=32 2024-12-04T14:11:34,727 INFO [RS:0;ec15ac6edd88:46401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321494727 2024-12-04T14:11:34,736 INFO [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321494727 2024-12-04T14:11:34,737 DEBUG [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:11:34,877 DEBUG [ec15ac6edd88:40617 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:11:34,882 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:34,887 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,46401,1733321493504, state=OPENING 2024-12-04T14:11:34,892 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:11:34,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:34,895 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:11:34,895 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:11:34,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ec15ac6edd88,46401,1733321493504}] 2024-12-04T14:11:35,071 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:35,073 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:11:35,076 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48352, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:11:35,089 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-04T14:11:35,090 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:11:35,093 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C46401%2C1733321493504.meta, suffix=.meta, logDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504, archiveDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs, maxLogs=32 2024-12-04T14:11:35,095 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.meta.1733321495094.meta 2024-12-04T14:11:35,103 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.meta.1733321495094.meta 2024-12-04T14:11:35,104 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:11:35,104 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:11:35,105 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:11:35,163 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:11:35,169 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:11:35,173 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:11:35,173 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:35,174 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-04T14:11:35,174 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-04T14:11:35,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:11:35,180 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:11:35,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:35,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:35,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:11:35,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:11:35,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:35,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:35,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:11:35,186 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:11:35,186 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:35,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:11:35,189 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740 2024-12-04T14:11:35,191 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740 2024-12-04T14:11:35,194 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:11:35,197 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:11:35,198 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=824834, jitterRate=0.048830851912498474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:11:35,200 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:11:35,207 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733321495066 2024-12-04T14:11:35,218 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:11:35,219 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-04T14:11:35,220 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:35,222 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,46401,1733321493504, state=OPEN 2024-12-04T14:11:35,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:11:35,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:11:35,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:11:35,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:11:35,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:11:35,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ec15ac6edd88,46401,1733321493504 in 331 msec 2024-12-04T14:11:35,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:11:35,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 768 msec 2024-12-04T14:11:35,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 942 msec 2024-12-04T14:11:35,244 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733321495244, completionTime=-1 2024-12-04T14:11:35,244 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:11:35,244 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-04T14:11:35,285 DEBUG [hconnection-0x5bfe2c9d-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:11:35,287 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48358, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:11:35,297 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-04T14:11:35,297 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733321555297 2024-12-04T14:11:35,297 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733321615297 2024-12-04T14:11:35,297 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 52 msec 2024-12-04T14:11:35,320 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40617,1733321492712-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:35,321 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40617,1733321492712-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:35,321 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40617,1733321492712-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:35,322 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec15ac6edd88:40617, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:35,323 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:35,328 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-04T14:11:35,331 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-04T14:11:35,333 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:11:35,339 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-04T14:11:35,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:11:35,343 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:35,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:11:35,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:11:35,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:11:35,361 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 08f2b64867fe585ceb7eda179c7d1375, NAME => 'hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78 2024-12-04T14:11:35,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:11:35,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:11:35,372 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:35,372 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 08f2b64867fe585ceb7eda179c7d1375, disabling compactions & flushes 2024-12-04T14:11:35,372 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,372 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,372 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. after waiting 0 ms 2024-12-04T14:11:35,372 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,372 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,372 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 08f2b64867fe585ceb7eda179c7d1375: 2024-12-04T14:11:35,375 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:11:35,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733321495376"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321495376"}]},"ts":"1733321495376"} 2024-12-04T14:11:35,405 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:11:35,407 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:11:35,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321495408"}]},"ts":"1733321495408"} 2024-12-04T14:11:35,415 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-04T14:11:35,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=08f2b64867fe585ceb7eda179c7d1375, ASSIGN}] 2024-12-04T14:11:35,423 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=08f2b64867fe585ceb7eda179c7d1375, ASSIGN 2024-12-04T14:11:35,425 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=08f2b64867fe585ceb7eda179c7d1375, ASSIGN; state=OFFLINE, location=ec15ac6edd88,46401,1733321493504; forceNewPlan=false, retain=false 2024-12-04T14:11:35,575 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=08f2b64867fe585ceb7eda179c7d1375, regionState=OPENING, regionLocation=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:35,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 08f2b64867fe585ceb7eda179c7d1375, server=ec15ac6edd88,46401,1733321493504}] 2024-12-04T14:11:35,734 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:35,741 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 08f2b64867fe585ceb7eda179c7d1375, NAME => 'hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:11:35,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:35,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,744 INFO [StoreOpener-08f2b64867fe585ceb7eda179c7d1375-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,747 INFO [StoreOpener-08f2b64867fe585ceb7eda179c7d1375-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08f2b64867fe585ceb7eda179c7d1375 columnFamilyName info 2024-12-04T14:11:35,747 DEBUG [StoreOpener-08f2b64867fe585ceb7eda179c7d1375-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:35,748 INFO [StoreOpener-08f2b64867fe585ceb7eda179c7d1375-1 {}] regionserver.HStore(327): Store=08f2b64867fe585ceb7eda179c7d1375/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:11:35,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,755 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:11:35,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:11:35,760 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 08f2b64867fe585ceb7eda179c7d1375; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773632, jitterRate=-0.016276374459266663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:11:35,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 08f2b64867fe585ceb7eda179c7d1375: 2024-12-04T14:11:35,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375., pid=6, masterSystemTime=1733321495734 2024-12-04T14:11:35,768 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,768 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:11:35,769 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=08f2b64867fe585ceb7eda179c7d1375, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:35,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:11:35,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 08f2b64867fe585ceb7eda179c7d1375, server=ec15ac6edd88,46401,1733321493504 in 192 msec 2024-12-04T14:11:35,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:11:35,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=08f2b64867fe585ceb7eda179c7d1375, ASSIGN in 356 msec 2024-12-04T14:11:35,781 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:11:35,781 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321495781"}]},"ts":"1733321495781"} 2024-12-04T14:11:35,784 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-04T14:11:35,788 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:11:35,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 455 msec 2024-12-04T14:11:35,843 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-04T14:11:35,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:11:35,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:35,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:11:35,875 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-04T14:11:35,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:11:35,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 25 msec 2024-12-04T14:11:35,909 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-04T14:11:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:11:35,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 17 msec 2024-12-04T14:11:35,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-04T14:11:35,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-04T14:11:35,938 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 2.335sec 2024-12-04T14:11:35,940 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:11:35,941 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:11:35,942 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:11:35,942 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:11:35,943 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:11:35,943 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40617,1733321492712-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:11:35,944 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40617,1733321492712-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:11:35,950 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:11:35,952 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:11:35,952 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40617,1733321492712-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:11:35,966 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c4612d8 to 127.0.0.1:63185 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@87fdff8 2024-12-04T14:11:35,966 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-04T14:11:35,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0c09d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:11:35,976 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T14:11:35,976 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T14:11:35,987 DEBUG [hconnection-0x3878e8d1-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:11:36,012 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48364, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:11:36,023 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ec15ac6edd88,40617,1733321492712 2024-12-04T14:11:36,023 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:11:36,031 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-04T14:11:36,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:11:36,041 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48528, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:11:36,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T14:11:36,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T14:11:36,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:11:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-12-04T14:11:36,056 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:11:36,057 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:36,059 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:11:36,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 9 2024-12-04T14:11:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:11:36,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741837_1013 (size=389) 2024-12-04T14:11:36,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741837_1013 (size=389) 2024-12-04T14:11:36,080 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8a6a87f086f16df5588841d1820da613, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78 2024-12-04T14:11:36,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741838_1014 (size=72) 2024-12-04T14:11:36,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741838_1014 (size=72) 2024-12-04T14:11:36,093 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:36,093 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 8a6a87f086f16df5588841d1820da613, disabling compactions & flushes 2024-12-04T14:11:36,093 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,093 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,093 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. after waiting 0 ms 2024-12-04T14:11:36,093 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,094 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,094 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:11:36,096 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:11:36,096 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733321496096"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321496096"}]},"ts":"1733321496096"} 2024-12-04T14:11:36,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:11:36,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:11:36,102 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321496102"}]},"ts":"1733321496102"} 2024-12-04T14:11:36,105 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-12-04T14:11:36,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8a6a87f086f16df5588841d1820da613, ASSIGN}] 2024-12-04T14:11:36,112 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8a6a87f086f16df5588841d1820da613, ASSIGN 2024-12-04T14:11:36,114 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8a6a87f086f16df5588841d1820da613, ASSIGN; state=OFFLINE, location=ec15ac6edd88,46401,1733321493504; forceNewPlan=false, retain=false 2024-12-04T14:11:36,265 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=8a6a87f086f16df5588841d1820da613, regionState=OPENING, regionLocation=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:36,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 8a6a87f086f16df5588841d1820da613, server=ec15ac6edd88,46401,1733321493504}] 2024-12-04T14:11:36,426 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:36,433 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,433 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 8a6a87f086f16df5588841d1820da613, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:11:36,434 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,434 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:11:36,434 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,434 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,437 INFO [StoreOpener-8a6a87f086f16df5588841d1820da613-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,439 INFO [StoreOpener-8a6a87f086f16df5588841d1820da613-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a6a87f086f16df5588841d1820da613 columnFamilyName info 2024-12-04T14:11:36,439 DEBUG [StoreOpener-8a6a87f086f16df5588841d1820da613-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:11:36,440 INFO [StoreOpener-8a6a87f086f16df5588841d1820da613-1 {}] regionserver.HStore(327): Store=8a6a87f086f16df5588841d1820da613/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:11:36,442 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,442 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,446 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:36,450 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:11:36,450 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 8a6a87f086f16df5588841d1820da613; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796470, jitterRate=0.012764021754264832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:11:36,452 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:11:36,453 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613., pid=11, masterSystemTime=1733321496426 2024-12-04T14:11:36,457 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,457 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:36,458 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=8a6a87f086f16df5588841d1820da613, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,46401,1733321493504 2024-12-04T14:11:36,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T14:11:36,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 8a6a87f086f16df5588841d1820da613, server=ec15ac6edd88,46401,1733321493504 in 191 msec 2024-12-04T14:11:36,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:11:36,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8a6a87f086f16df5588841d1820da613, ASSIGN in 356 msec 2024-12-04T14:11:36,471 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:11:36,472 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321496471"}]},"ts":"1733321496471"} 2024-12-04T14:11:36,475 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-12-04T14:11:36,479 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:11:36,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 427 msec 2024-12-04T14:11:40,678 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:11:40,717 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T14:11:40,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-04T14:11:40,719 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-12-04T14:11:43,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T14:11:43,240 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T14:11:43,242 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-04T14:11:43,242 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T14:11:43,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-04T14:11:43,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-04T14:11:43,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:11:43,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T14:11:43,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T14:11:43,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T14:11:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40617 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:11:46,072 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling, procId: 9 completed 2024-12-04T14:11:46,076 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-12-04T14:11:46,078 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:11:46,079 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321506078 2024-12-04T14:11:46,089 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321494727 with entries=4, filesize=947 B; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321506078 2024-12-04T14:11:46,090 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45005:45005),(127.0.0.1/127.0.0.1:44197:44197)] 2024-12-04T14:11:46,090 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321494727 is not closed yet, will try archiving it next time 2024-12-04T14:11:46,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741833_1009 (size=955) 2024-12-04T14:11:46,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741833_1009 (size=955) 2024-12-04T14:11:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] regionserver.HRegion(8581): Flush requested on 8a6a87f086f16df5588841d1820da613 2024-12-04T14:11:58,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8a6a87f086f16df5588841d1820da613 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:11:58,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/bd0b8475a9ab44e8ba90986f8113bf6c is 1080, key is row0001/info:/1733321506096/Put/seqid=0 2024-12-04T14:11:58,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741840_1016 (size=12509) 2024-12-04T14:11:58,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741840_1016 (size=12509) 2024-12-04T14:11:58,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/bd0b8475a9ab44e8ba90986f8113bf6c 2024-12-04T14:11:58,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/bd0b8475a9ab44e8ba90986f8113bf6c as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c 2024-12-04T14:11:58,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c, entries=7, sequenceid=11, filesize=12.2 K 2024-12-04T14:11:58,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8a6a87f086f16df5588841d1820da613 in 144ms, sequenceid=11, compaction requested=false 2024-12-04T14:11:58,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:12:01,791 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:12:06,133 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321526132 2024-12-04T14:12:06,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:06,342 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321506078 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321526132 2024-12-04T14:12:06,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45005:45005),(127.0.0.1/127.0.0.1:44197:44197)] 2024-12-04T14:12:06,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321506078 is not closed yet, will try archiving it next time 2024-12-04T14:12:06,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741839_1015 (size=12399) 2024-12-04T14:12:06,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741839_1015 (size=12399) 2024-12-04T14:12:06,381 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:12:06,383 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59954, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:12:06,545 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:08,749 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:10,952 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:13,156 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:13,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] regionserver.HRegion(8581): Flush requested on 8a6a87f086f16df5588841d1820da613 2024-12-04T14:12:13,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8a6a87f086f16df5588841d1820da613 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:12:13,358 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:13,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/bf198b18b35f46e69458448d5a8226ea is 1080, key is row0008/info:/1733321520124/Put/seqid=0 2024-12-04T14:12:13,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741842_1018 (size=12509) 2024-12-04T14:12:13,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741842_1018 (size=12509) 2024-12-04T14:12:13,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/bf198b18b35f46e69458448d5a8226ea 2024-12-04T14:12:13,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/bf198b18b35f46e69458448d5a8226ea as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bf198b18b35f46e69458448d5a8226ea 2024-12-04T14:12:13,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bf198b18b35f46e69458448d5a8226ea, entries=7, sequenceid=21, filesize=12.2 K 2024-12-04T14:12:13,612 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:13,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8a6a87f086f16df5588841d1820da613 in 456ms, sequenceid=21, compaction requested=false 2024-12-04T14:12:13,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:12:13,613 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=24.4 K, sizeToCheck=16.0 K 2024-12-04T14:12:13,613 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:12:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c because midkey is the same as first or last row 2024-12-04T14:12:15,360 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:15,953 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:12:15,953 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:12:17,564 WARN [sync.1 {}] wal.AbstractFSWAL(1346): Requesting log roll because we exceeded slow sync threshold; count=7, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:17,565 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C46401%2C1733321493504:(num 1733321526132) roll requested 2024-12-04T14:12:17,565 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:17,565 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321537565 2024-12-04T14:12:17,774 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:17,974 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK]] 2024-12-04T14:12:17,976 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321526132 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321537565 2024-12-04T14:12:17,976 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:12:17,976 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321526132 is not closed yet, will try archiving it next time 2024-12-04T14:12:17,977 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321506078 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321506078 2024-12-04T14:12:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741841_1017 (size=7739) 2024-12-04T14:12:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741841_1017 (size=7739) 2024-12-04T14:12:19,767 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:21,434 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 8a6a87f086f16df5588841d1820da613, had cached 0 bytes from a total of 25018 2024-12-04T14:12:21,970 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:24,174 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:26,378 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:28,380 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T14:12:28,380 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321548380 2024-12-04T14:12:31,791 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:12:33,390 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:33,390 WARN [Time-limited test {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:33,390 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C46401%2C1733321493504:(num 1733321548380) roll requested 2024-12-04T14:12:35,322 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 08f2b64867fe585ceb7eda179c7d1375 changed from -1.0 to 0.0, refreshing cache 2024-12-04T14:12:38,390 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:38,390 WARN [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:38,391 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321537565 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321548380 2024-12-04T14:12:38,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:12:38,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321537565 is not closed yet, will try archiving it next time 2024-12-04T14:12:38,392 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321558392 2024-12-04T14:12:38,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741843_1019 (size=4753) 2024-12-04T14:12:38,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741843_1019 (size=4753) 2024-12-04T14:12:43,394 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:43,394 WARN [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] regionserver.HRegion(8581): Flush requested on 8a6a87f086f16df5588841d1820da613 2024-12-04T14:12:43,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8a6a87f086f16df5588841d1820da613 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:12:43,400 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:43,400 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:45,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T14:12:48,396 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:48,396 WARN [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:48,400 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:48,400 WARN [sync.0 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:48,402 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321548380 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321558392 2024-12-04T14:12:48,402 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:12:48,402 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321548380 is not closed yet, will try archiving it next time 2024-12-04T14:12:48,402 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C46401%2C1733321493504:(num 1733321558392) roll requested 2024-12-04T14:12:48,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/05f23de6e333438784f99de4e6651a34 is 1080, key is row0015/info:/1733321535158/Put/seqid=0 2024-12-04T14:12:48,403 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321568402 2024-12-04T14:12:48,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741844_1020 (size=1569) 2024-12-04T14:12:48,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741844_1020 (size=1569) 2024-12-04T14:12:48,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741846_1022 (size=12509) 2024-12-04T14:12:48,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741846_1022 (size=12509) 2024-12-04T14:12:48,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/05f23de6e333438784f99de4e6651a34 2024-12-04T14:12:48,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/05f23de6e333438784f99de4e6651a34 as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/05f23de6e333438784f99de4e6651a34 2024-12-04T14:12:48,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/05f23de6e333438784f99de4e6651a34, entries=7, sequenceid=31, filesize=12.2 K 2024-12-04T14:12:53,411 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:53,411 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:53,438 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:53,438 WARN [sync.1 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:53,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8a6a87f086f16df5588841d1820da613 in 10044ms, sequenceid=31, compaction requested=true 2024-12-04T14:12:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:12:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=36.6 K, sizeToCheck=16.0 K 2024-12-04T14:12:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:12:53,439 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c because midkey is the same as first or last row 2024-12-04T14:12:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a6a87f086f16df5588841d1820da613:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:12:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:12:53,441 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:12:53,444 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:12:53,445 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.HStore(1540): 8a6a87f086f16df5588841d1820da613/info is initiating minor compaction (all files) 2024-12-04T14:12:53,445 INFO [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8a6a87f086f16df5588841d1820da613/info in TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:12:53,446 INFO [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bf198b18b35f46e69458448d5a8226ea, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/05f23de6e333438784f99de4e6651a34] into tmpdir=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp, totalSize=36.6 K 2024-12-04T14:12:53,447 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] compactions.Compactor(224): Compacting bd0b8475a9ab44e8ba90986f8113bf6c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733321506096 2024-12-04T14:12:53,447 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] compactions.Compactor(224): Compacting bf198b18b35f46e69458448d5a8226ea, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733321520124 2024-12-04T14:12:53,448 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] compactions.Compactor(224): Compacting 05f23de6e333438784f99de4e6651a34, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733321535158 2024-12-04T14:12:53,488 INFO [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a6a87f086f16df5588841d1820da613#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:12:53,489 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/f9cad533ace54217b6041270c28d7e4b is 1080, key is row0001/info:/1733321506096/Put/seqid=0 2024-12-04T14:12:53,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741848_1024 (size=27710) 2024-12-04T14:12:53,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741848_1024 (size=27710) 2024-12-04T14:12:53,536 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/f9cad533ace54217b6041270c28d7e4b as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/f9cad533ace54217b6041270c28d7e4b 2024-12-04T14:12:58,412 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:58,412 WARN [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:12:58,413 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321558392 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321568402 2024-12-04T14:12:58,413 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:12:58,413 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321558392 is not closed yet, will try archiving it next time 2024-12-04T14:12:58,413 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321526132 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321526132 2024-12-04T14:12:58,413 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C46401%2C1733321493504:(num 1733321578413) roll requested 2024-12-04T14:12:58,413 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321578413 2024-12-04T14:12:58,415 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321537565 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321537565 2024-12-04T14:12:58,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741845_1021 (size=438) 2024-12-04T14:12:58,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741845_1021 (size=438) 2024-12-04T14:12:58,417 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321548380 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321548380 2024-12-04T14:12:58,418 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321558392 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321558392 2024-12-04T14:13:01,791 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:13:03,414 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:13:03,414 WARN [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:13:03,416 INFO [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8a6a87f086f16df5588841d1820da613/info of 8a6a87f086f16df5588841d1820da613 into f9cad533ace54217b6041270c28d7e4b(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 9sec to execute. 2024-12-04T14:13:03,416 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:13:03,416 INFO [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613., storeName=8a6a87f086f16df5588841d1820da613/info, priority=13, startTime=1733321573440; duration=9sec 2024-12-04T14:13:03,417 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=27.1 K, sizeToCheck=16.0 K 2024-12-04T14:13:03,417 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:13:03,417 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/f9cad533ace54217b6041270c28d7e4b because midkey is the same as first or last row 2024-12-04T14:13:03,417 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:13:03,417 DEBUG [RS:0;ec15ac6edd88:46401-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a6a87f086f16df5588841d1820da613:info 2024-12-04T14:13:03,422 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:13:03,422 WARN [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46499,DS-929b6647-d0dd-46dc-b58d-4c259bd3016f,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-c7a2420d-549a-4acb-b2c4-c23cebeaf0ee,DISK]] 2024-12-04T14:13:03,423 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321568402 with entries=1, filesize=531 B; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321578413 2024-12-04T14:13:03,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:13:03,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321568402 is not closed yet, will try archiving it next time 2024-12-04T14:13:03,423 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321583423 2024-12-04T14:13:03,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741847_1023 (size=539) 2024-12-04T14:13:03,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741847_1023 (size=539) 2024-12-04T14:13:03,427 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321568402 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321568402 2024-12-04T14:13:03,433 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321578413 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321583423 2024-12-04T14:13:03,433 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:13:03,433 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321578413 is not closed yet, will try archiving it next time 2024-12-04T14:13:03,433 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C46401%2C1733321493504:(num 1733321583423) roll requested 2024-12-04T14:13:03,433 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321493504.1733321583433 2024-12-04T14:13:03,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741849_1025 (size=1258) 2024-12-04T14:13:03,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741849_1025 (size=1258) 2024-12-04T14:13:03,442 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321583423 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321583433 2024-12-04T14:13:03,443 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44197:44197),(127.0.0.1/127.0.0.1:45005:45005)] 2024-12-04T14:13:03,443 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321583423 is not closed yet, will try archiving it next time 2024-12-04T14:13:03,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741850_1026 (size=93) 2024-12-04T14:13:03,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741850_1026 (size=93) 2024-12-04T14:13:03,445 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504/ec15ac6edd88%2C46401%2C1733321493504.1733321583423 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs/ec15ac6edd88%2C46401%2C1733321493504.1733321583423 2024-12-04T14:13:06,435 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 8a6a87f086f16df5588841d1820da613, had cached 0 bytes from a total of 27710 2024-12-04T14:13:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] regionserver.HRegion(8581): Flush requested on 8a6a87f086f16df5588841d1820da613 2024-12-04T14:13:15,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8a6a87f086f16df5588841d1820da613 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:13:15,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/e1982d3a143c4da8ae0997d4c7eb157a is 1080, key is row0022/info:/1733321583424/Put/seqid=0 2024-12-04T14:13:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741852_1028 (size=12509) 2024-12-04T14:13:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741852_1028 (size=12509) 2024-12-04T14:13:15,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/e1982d3a143c4da8ae0997d4c7eb157a 2024-12-04T14:13:15,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/e1982d3a143c4da8ae0997d4c7eb157a as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/e1982d3a143c4da8ae0997d4c7eb157a 2024-12-04T14:13:15,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/e1982d3a143c4da8ae0997d4c7eb157a, entries=7, sequenceid=42, filesize=12.2 K 2024-12-04T14:13:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8a6a87f086f16df5588841d1820da613 in 34ms, sequenceid=42, compaction requested=false 2024-12-04T14:13:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:13:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=39.3 K, sizeToCheck=16.0 K 2024-12-04T14:13:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:13:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/f9cad533ace54217b6041270c28d7e4b because midkey is the same as first or last row 2024-12-04T14:13:23,452 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-04T14:13:23,453 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-04T14:13:23,453 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c4612d8 to 127.0.0.1:63185 2024-12-04T14:13:23,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:13:23,454 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:13:23,454 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1895866917, stopped=false 2024-12-04T14:13:23,454 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ec15ac6edd88,40617,1733321492712 2024-12-04T14:13:23,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:13:23,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:13:23,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:23,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-04T14:13:23,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:23,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:13:23,460 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,46401,1733321493504' ***** 2024-12-04T14:13:23,460 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:13:23,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:13:23,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:13:23,460 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:13:23,460 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:13:23,461 INFO [RS:0;ec15ac6edd88:46401 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:13:23,461 INFO [RS:0;ec15ac6edd88:46401 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:13:23,461 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(3579): Received CLOSE for 08f2b64867fe585ceb7eda179c7d1375 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(3579): Received CLOSE for 8a6a87f086f16df5588841d1820da613 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,46401,1733321493504 2024-12-04T14:13:23,462 DEBUG [RS:0;ec15ac6edd88:46401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 08f2b64867fe585ceb7eda179c7d1375, disabling compactions & flushes 2024-12-04T14:13:23,462 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:13:23,462 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:13:23,462 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 08f2b64867fe585ceb7eda179c7d1375=hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375., 8a6a87f086f16df5588841d1820da613=TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.} 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. after waiting 0 ms 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:13:23,462 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 08f2b64867fe585ceb7eda179c7d1375 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-04T14:13:23,462 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:13:23,462 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:13:23,463 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.81 KB heapSize=5.32 KB 2024-12-04T14:13:23,463 DEBUG [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1629): Waiting on 08f2b64867fe585ceb7eda179c7d1375, 1588230740, 8a6a87f086f16df5588841d1820da613 2024-12-04T14:13:23,480 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/.tmp/info/58663095be53463fbe690c9c9ba48c46 is 45, key is default/info:d/1733321495885/Put/seqid=0 2024-12-04T14:13:23,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741853_1029 (size=5037) 2024-12-04T14:13:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741853_1029 (size=5037) 2024-12-04T14:13:23,490 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/.tmp/info/2927bed2faab43d385d7889f900b7c32 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613./info:regioninfo/1733321496458/Put/seqid=0 2024-12-04T14:13:23,491 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/.tmp/info/58663095be53463fbe690c9c9ba48c46 2024-12-04T14:13:23,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741854_1030 (size=8172) 2024-12-04T14:13:23,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741854_1030 (size=8172) 2024-12-04T14:13:23,502 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.59 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/.tmp/info/2927bed2faab43d385d7889f900b7c32 2024-12-04T14:13:23,504 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/.tmp/info/58663095be53463fbe690c9c9ba48c46 as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/info/58663095be53463fbe690c9c9ba48c46 2024-12-04T14:13:23,513 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/info/58663095be53463fbe690c9c9ba48c46, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T14:13:23,514 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 08f2b64867fe585ceb7eda179c7d1375 in 52ms, sequenceid=6, compaction requested=false 2024-12-04T14:13:23,520 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/namespace/08f2b64867fe585ceb7eda179c7d1375/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T14:13:23,523 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:13:23,523 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 08f2b64867fe585ceb7eda179c7d1375: 2024-12-04T14:13:23,523 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733321495332.08f2b64867fe585ceb7eda179c7d1375. 2024-12-04T14:13:23,524 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8a6a87f086f16df5588841d1820da613, disabling compactions & flushes 2024-12-04T14:13:23,524 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:13:23,524 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:13:23,524 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. after waiting 0 ms 2024-12-04T14:13:23,524 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:13:23,524 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8a6a87f086f16df5588841d1820da613 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-12-04T14:13:23,527 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/.tmp/table/aa33d5c6dce74162beb3ef68c011663f is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733321496471/Put/seqid=0 2024-12-04T14:13:23,529 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/7d8b6c8def274ec7a0bee156829515ff is 1080, key is row0029/info:/1733321597445/Put/seqid=0 2024-12-04T14:13:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741855_1031 (size=5452) 2024-12-04T14:13:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741855_1031 (size=5452) 2024-12-04T14:13:23,534 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=232 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/.tmp/table/aa33d5c6dce74162beb3ef68c011663f 2024-12-04T14:13:23,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741856_1032 (size=8193) 2024-12-04T14:13:23,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741856_1032 (size=8193) 2024-12-04T14:13:23,540 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/7d8b6c8def274ec7a0bee156829515ff 2024-12-04T14:13:23,543 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/.tmp/info/2927bed2faab43d385d7889f900b7c32 as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/info/2927bed2faab43d385d7889f900b7c32 2024-12-04T14:13:23,548 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/.tmp/info/7d8b6c8def274ec7a0bee156829515ff as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/7d8b6c8def274ec7a0bee156829515ff 2024-12-04T14:13:23,551 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/info/2927bed2faab43d385d7889f900b7c32, entries=20, sequenceid=14, filesize=8.0 K 2024-12-04T14:13:23,552 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/.tmp/table/aa33d5c6dce74162beb3ef68c011663f as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/table/aa33d5c6dce74162beb3ef68c011663f 2024-12-04T14:13:23,556 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/7d8b6c8def274ec7a0bee156829515ff, entries=3, sequenceid=48, filesize=8.0 K 2024-12-04T14:13:23,557 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 8a6a87f086f16df5588841d1820da613 in 33ms, sequenceid=48, compaction requested=true 2024-12-04T14:13:23,557 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bf198b18b35f46e69458448d5a8226ea, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/05f23de6e333438784f99de4e6651a34] to archive 2024-12-04T14:13:23,560 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/table/aa33d5c6dce74162beb3ef68c011663f, entries=4, sequenceid=14, filesize=5.3 K 2024-12-04T14:13:23,561 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:13:23,562 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.81 KB/2882, heapSize ~5.04 KB/5160, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=14, compaction requested=false 2024-12-04T14:13:23,564 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/archive/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bd0b8475a9ab44e8ba90986f8113bf6c 2024-12-04T14:13:23,566 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bf198b18b35f46e69458448d5a8226ea to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/archive/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/bf198b18b35f46e69458448d5a8226ea 2024-12-04T14:13:23,568 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1 2024-12-04T14:13:23,568 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/05f23de6e333438784f99de4e6651a34 to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/archive/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/info/05f23de6e333438784f99de4e6651a34 2024-12-04T14:13:23,568 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:13:23,569 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:13:23,569 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:13:23,569 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T14:13:23,572 INFO [regionserver/ec15ac6edd88:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T14:13:23,572 INFO [regionserver/ec15ac6edd88:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T14:13:23,587 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/data/default/TestLogRolling-testSlowSyncLogRolling/8a6a87f086f16df5588841d1820da613/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-12-04T14:13:23,588 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:13:23,588 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8a6a87f086f16df5588841d1820da613: 2024-12-04T14:13:23,588 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733321496047.8a6a87f086f16df5588841d1820da613. 2024-12-04T14:13:23,663 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,46401,1733321493504; all regions closed. 2024-12-04T14:13:23,664 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504 2024-12-04T14:13:23,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741834_1010 (size=4330) 2024-12-04T14:13:23,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741834_1010 (size=4330) 2024-12-04T14:13:23,670 DEBUG [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs 2024-12-04T14:13:23,670 INFO [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C46401%2C1733321493504.meta:.meta(num 1733321495094) 2024-12-04T14:13:23,670 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/WALs/ec15ac6edd88,46401,1733321493504 2024-12-04T14:13:23,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741851_1027 (size=13066) 2024-12-04T14:13:23,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741851_1027 (size=13066) 2024-12-04T14:13:23,677 DEBUG [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/oldWALs 2024-12-04T14:13:23,677 INFO [RS:0;ec15ac6edd88:46401 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C46401%2C1733321493504:(num 1733321583433) 2024-12-04T14:13:23,677 DEBUG [RS:0;ec15ac6edd88:46401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:13:23,677 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:13:23,677 INFO [RS:0;ec15ac6edd88:46401 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-04T14:13:23,678 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:13:23,678 INFO [RS:0;ec15ac6edd88:46401 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46401 2024-12-04T14:13:23,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,46401,1733321493504 2024-12-04T14:13:23,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:13:23,684 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,46401,1733321493504] 2024-12-04T14:13:23,684 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,46401,1733321493504; numProcessing=1 2024-12-04T14:13:23,686 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,46401,1733321493504 already deleted, retry=false 2024-12-04T14:13:23,686 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,46401,1733321493504 expired; onlineServers=0 2024-12-04T14:13:23,686 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,40617,1733321492712' ***** 2024-12-04T14:13:23,686 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:13:23,686 DEBUG [M:0;ec15ac6edd88:40617 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7042fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:13:23,686 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,40617,1733321492712 2024-12-04T14:13:23,687 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,40617,1733321492712; all regions closed. 2024-12-04T14:13:23,687 DEBUG [M:0;ec15ac6edd88:40617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:13:23,687 DEBUG [M:0;ec15ac6edd88:40617 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:13:23,687 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:13:23,687 DEBUG [M:0;ec15ac6edd88:40617 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:13:23,687 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321494380 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321494380,5,FailOnTimeoutGroup] 2024-12-04T14:13:23,687 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321494382 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321494382,5,FailOnTimeoutGroup] 2024-12-04T14:13:23,687 INFO [M:0;ec15ac6edd88:40617 {}] hbase.ChoreService(370): Chore service for: master/ec15ac6edd88:0 had [] on shutdown 2024-12-04T14:13:23,687 DEBUG [M:0;ec15ac6edd88:40617 {}] master.HMaster(1733): Stopping service threads 2024-12-04T14:13:23,687 INFO [M:0;ec15ac6edd88:40617 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:13:23,688 INFO [M:0;ec15ac6edd88:40617 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:13:23,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:13:23,688 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:13:23,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:23,689 DEBUG [M:0;ec15ac6edd88:40617 {}] zookeeper.ZKUtil(347): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:13:23,689 WARN [M:0;ec15ac6edd88:40617 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:13:23,689 INFO [M:0;ec15ac6edd88:40617 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-04T14:13:23,689 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:13:23,689 INFO [M:0;ec15ac6edd88:40617 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:13:23,689 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:13:23,689 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:23,689 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:23,689 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:13:23,689 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:23,690 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.20 KB heapSize=50.12 KB 2024-12-04T14:13:23,709 DEBUG [M:0;ec15ac6edd88:40617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ea33f5038f7543fda9d2b5666de09b2b is 82, key is hbase:meta,,1/info:regioninfo/1733321495219/Put/seqid=0 2024-12-04T14:13:23,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741857_1033 (size=5672) 2024-12-04T14:13:23,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741857_1033 (size=5672) 2024-12-04T14:13:23,716 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ea33f5038f7543fda9d2b5666de09b2b 2024-12-04T14:13:23,739 DEBUG [M:0;ec15ac6edd88:40617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d673eae357fb460ea3311d00c575b419 is 765, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733321496481/Put/seqid=0 2024-12-04T14:13:23,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741858_1034 (size=6425) 2024-12-04T14:13:23,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741858_1034 (size=6425) 2024-12-04T14:13:23,746 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.59 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d673eae357fb460ea3311d00c575b419 2024-12-04T14:13:23,753 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d673eae357fb460ea3311d00c575b419 2024-12-04T14:13:23,775 DEBUG [M:0;ec15ac6edd88:40617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f17a50c867824c9f906464f1c63750e7 is 69, key is ec15ac6edd88,46401,1733321493504/rs:state/1733321494496/Put/seqid=0 2024-12-04T14:13:23,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741859_1035 (size=5156) 2024-12-04T14:13:23,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741859_1035 (size=5156) 2024-12-04T14:13:23,782 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f17a50c867824c9f906464f1c63750e7 2024-12-04T14:13:23,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:13:23,784 INFO [RS:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,46401,1733321493504; zookeeper connection closed. 2024-12-04T14:13:23,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46401-0x1005d5ebe780001, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:13:23,784 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@8f3f92a {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@8f3f92a 2024-12-04T14:13:23,785 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T14:13:23,812 DEBUG [M:0;ec15ac6edd88:40617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6c07845170e9440a830e9525f09bb1a4 is 52, key is load_balancer_on/state:d/1733321496028/Put/seqid=0 2024-12-04T14:13:23,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741860_1036 (size=5056) 2024-12-04T14:13:23,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741860_1036 (size=5056) 2024-12-04T14:13:23,819 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6c07845170e9440a830e9525f09bb1a4 2024-12-04T14:13:23,826 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ea33f5038f7543fda9d2b5666de09b2b as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ea33f5038f7543fda9d2b5666de09b2b 2024-12-04T14:13:23,833 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ea33f5038f7543fda9d2b5666de09b2b, entries=8, sequenceid=104, filesize=5.5 K 2024-12-04T14:13:23,834 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d673eae357fb460ea3311d00c575b419 as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d673eae357fb460ea3311d00c575b419 2024-12-04T14:13:23,839 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d673eae357fb460ea3311d00c575b419 2024-12-04T14:13:23,839 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d673eae357fb460ea3311d00c575b419, entries=11, sequenceid=104, filesize=6.3 K 2024-12-04T14:13:23,841 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f17a50c867824c9f906464f1c63750e7 as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f17a50c867824c9f906464f1c63750e7 2024-12-04T14:13:23,846 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f17a50c867824c9f906464f1c63750e7, entries=1, sequenceid=104, filesize=5.0 K 2024-12-04T14:13:23,847 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6c07845170e9440a830e9525f09bb1a4 as hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6c07845170e9440a830e9525f09bb1a4 2024-12-04T14:13:23,853 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6c07845170e9440a830e9525f09bb1a4, entries=1, sequenceid=104, filesize=4.9 K 2024-12-04T14:13:23,854 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.20 KB/41161, heapSize ~50.05 KB/51256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=104, compaction requested=false 2024-12-04T14:13:23,856 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:23,856 DEBUG [M:0;ec15ac6edd88:40617 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:13:23,856 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/MasterData/WALs/ec15ac6edd88,40617,1733321492712 2024-12-04T14:13:23,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741830_1006 (size=48462) 2024-12-04T14:13:23,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741830_1006 (size=48462) 2024-12-04T14:13:23,859 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:13:23,859 INFO [M:0;ec15ac6edd88:40617 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-04T14:13:23,859 INFO [M:0;ec15ac6edd88:40617 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40617 2024-12-04T14:13:23,861 DEBUG [M:0;ec15ac6edd88:40617 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ec15ac6edd88,40617,1733321492712 already deleted, retry=false 2024-12-04T14:13:23,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:13:23,963 INFO [M:0;ec15ac6edd88:40617 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,40617,1733321492712; zookeeper connection closed. 2024-12-04T14:13:23,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40617-0x1005d5ebe780000, quorum=127.0.0.1:63185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:13:23,968 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@163cfad6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:23,970 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:23,970 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:23,971 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:23,971 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:23,974 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:13:23,974 WARN [BP-802734383-172.17.0.2-1733321489678 heartbeating to localhost/127.0.0.1:37417 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:13:23,974 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:13:23,974 WARN [BP-802734383-172.17.0.2-1733321489678 heartbeating to localhost/127.0.0.1:37417 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-802734383-172.17.0.2-1733321489678 (Datanode Uuid e1aa4844-70c0-46cc-ae59-f5d904479534) service to localhost/127.0.0.1:37417 2024-12-04T14:13:23,975 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data3/current/BP-802734383-172.17.0.2-1733321489678 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:23,976 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data4/current/BP-802734383-172.17.0.2-1733321489678 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:23,976 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:13:23,981 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6aad8790{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:23,981 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:23,981 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:23,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:23,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:23,984 WARN [BP-802734383-172.17.0.2-1733321489678 heartbeating to localhost/127.0.0.1:37417 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:13:23,984 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:13:23,984 WARN [BP-802734383-172.17.0.2-1733321489678 heartbeating to localhost/127.0.0.1:37417 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-802734383-172.17.0.2-1733321489678 (Datanode Uuid 3180f54a-891e-48b6-866d-0ef0be60ea1e) service to localhost/127.0.0.1:37417 2024-12-04T14:13:23,984 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:13:23,985 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data1/current/BP-802734383-172.17.0.2-1733321489678 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:23,985 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/cluster_5a395645-3752-b692-9f51-8947f7d03dcf/dfs/data/data2/current/BP-802734383-172.17.0.2-1733321489678 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:23,985 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:13:23,993 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5682c4d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:13:23,994 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:23,994 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:23,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:23,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:24,006 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:13:24,046 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-04T14:13:24,055 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=62 (was 12) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37417 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: master/ec15ac6edd88:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ec15ac6edd88:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:37417 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37417 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ec15ac6edd88:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37417 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37417 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37417 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@4dc21135 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37417 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ec15ac6edd88:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37417 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=406 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=259 (was 301), ProcessCount=11 (was 11), AvailableMemoryMB=6635 (was 7205) 2024-12-04T14:13:24,062 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=63, OpenFileDescriptor=406, MaxFileDescriptor=1048576, SystemLoadAverage=259, ProcessCount=11, AvailableMemoryMB=6634 2024-12-04T14:13:24,062 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.log.dir so I do NOT create it in target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9593c776-35f7-e92d-5024-208470c2bd70/hadoop.tmp.dir so I do NOT create it in target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567, deleteOnExit=true 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/test.cache.data in system properties and HBase conf 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:13:24,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-04T14:13:24,063 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:13:24,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:13:24,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:13:24,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:13:24,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:13:24,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:13:24,079 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:13:24,147 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:24,152 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:24,154 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:24,154 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:24,155 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:13:24,159 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:24,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25d9b335{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:24,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4063a7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:24,279 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@654852b8{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-38019-hadoop-hdfs-3_4_1-tests_jar-_-any-15811997745886372/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:13:24,280 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74e82574{HTTP/1.1, (http/1.1)}{localhost:38019} 2024-12-04T14:13:24,280 INFO [Time-limited test {}] server.Server(415): Started @116750ms 2024-12-04T14:13:24,295 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:13:24,374 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:24,379 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:24,381 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:24,381 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:24,381 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:13:24,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ad3a453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:24,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64f9fde3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:24,508 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a43de3a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-45529-hadoop-hdfs-3_4_1-tests_jar-_-any-18335519654199562830/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:24,508 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@183f70b1{HTTP/1.1, (http/1.1)}{localhost:45529} 2024-12-04T14:13:24,508 INFO [Time-limited test {}] server.Server(415): Started @116979ms 2024-12-04T14:13:24,510 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:13:24,548 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:24,554 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:24,557 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:24,557 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:24,557 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:13:24,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e043b3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:24,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f686eed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:24,577 INFO [regionserver/ec15ac6edd88:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:13:24,610 WARN [Thread-450 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data1/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:24,610 WARN [Thread-451 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data2/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:24,629 WARN [Thread-429 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:13:24,634 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x108eee87484447c with lease ID 0xb59c7031934b02d5: Processing first storage report for DS-58a778b9-546d-4598-9fc0-013c034938d5 from datanode DatanodeRegistration(127.0.0.1:46075, datanodeUuid=c78f2261-d155-4e80-b7c4-fb67ad306bcc, infoPort=36305, infoSecurePort=0, ipcPort=40497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:24,634 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x108eee87484447c with lease ID 0xb59c7031934b02d5: from storage DS-58a778b9-546d-4598-9fc0-013c034938d5 node DatanodeRegistration(127.0.0.1:46075, datanodeUuid=c78f2261-d155-4e80-b7c4-fb67ad306bcc, infoPort=36305, infoSecurePort=0, ipcPort=40497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:24,634 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x108eee87484447c with lease ID 0xb59c7031934b02d5: Processing first storage report for DS-df9338b7-f17e-415c-897f-caeb630c6604 from datanode DatanodeRegistration(127.0.0.1:46075, datanodeUuid=c78f2261-d155-4e80-b7c4-fb67ad306bcc, infoPort=36305, infoSecurePort=0, ipcPort=40497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:24,634 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x108eee87484447c with lease ID 0xb59c7031934b02d5: from storage DS-df9338b7-f17e-415c-897f-caeb630c6604 node DatanodeRegistration(127.0.0.1:46075, datanodeUuid=c78f2261-d155-4e80-b7c4-fb67ad306bcc, infoPort=36305, infoSecurePort=0, ipcPort=40497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:24,684 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3d3898a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-33273-hadoop-hdfs-3_4_1-tests_jar-_-any-790535740779436034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:24,685 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e8dad2b{HTTP/1.1, (http/1.1)}{localhost:33273} 2024-12-04T14:13:24,685 INFO [Time-limited test {}] server.Server(415): Started @117155ms 2024-12-04T14:13:24,686 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:13:24,772 WARN [Thread-477 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data4/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:24,772 WARN [Thread-476 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data3/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:24,799 WARN [Thread-465 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:13:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcbef2e4309666003 with lease ID 0xb59c7031934b02d6: Processing first storage report for DS-9222f14d-c6ad-48a7-be72-9411e899a952 from datanode DatanodeRegistration(127.0.0.1:35023, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38609, infoSecurePort=0, ipcPort=38939, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcbef2e4309666003 with lease ID 0xb59c7031934b02d6: from storage DS-9222f14d-c6ad-48a7-be72-9411e899a952 node DatanodeRegistration(127.0.0.1:35023, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38609, infoSecurePort=0, ipcPort=38939, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:13:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcbef2e4309666003 with lease ID 0xb59c7031934b02d6: Processing first storage report for DS-d3639750-1cab-43fc-8d86-d75999ae8b8f from datanode DatanodeRegistration(127.0.0.1:35023, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38609, infoSecurePort=0, ipcPort=38939, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcbef2e4309666003 with lease ID 0xb59c7031934b02d6: from storage DS-d3639750-1cab-43fc-8d86-d75999ae8b8f node DatanodeRegistration(127.0.0.1:35023, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38609, infoSecurePort=0, ipcPort=38939, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:24,825 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30 2024-12-04T14:13:24,828 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/zookeeper_0, clientPort=52543, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:13:24,829 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=52543 2024-12-04T14:13:24,829 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,831 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:13:24,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:13:24,844 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8 with version=8 2024-12-04T14:13:24,844 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase-staging 2024-12-04T14:13:24,847 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:13:24,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:24,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:24,847 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:13:24,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:24,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:13:24,847 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:13:24,848 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:13:24,848 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46261 2024-12-04T14:13:24,849 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,850 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,853 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:46261 connecting to ZooKeeper ensemble=127.0.0.1:52543 2024-12-04T14:13:24,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462610x0, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:13:24,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46261-0x1005d6077c20000 connected 2024-12-04T14:13:24,874 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:13:24,875 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:13:24,876 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:13:24,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46261 2024-12-04T14:13:24,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46261 2024-12-04T14:13:24,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46261 2024-12-04T14:13:24,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46261 2024-12-04T14:13:24,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46261 2024-12-04T14:13:24,877 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8, hbase.cluster.distributed=false 2024-12-04T14:13:24,895 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:13:24,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:24,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:24,896 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:13:24,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:24,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:13:24,896 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:13:24,896 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:13:24,897 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40113 2024-12-04T14:13:24,897 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:13:24,900 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:13:24,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,903 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,906 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:40113 connecting to ZooKeeper ensemble=127.0.0.1:52543 2024-12-04T14:13:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401130x0, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:13:24,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:13:24,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40113-0x1005d6077c20001 connected 2024-12-04T14:13:24,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:13:24,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:13:24,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40113 2024-12-04T14:13:24,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40113 2024-12-04T14:13:24,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40113 2024-12-04T14:13:24,922 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40113 2024-12-04T14:13:24,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40113 2024-12-04T14:13:24,925 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:24,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:13:24,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:13:24,927 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:24,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:13:24,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:24,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:13:24,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:24,932 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:13:24,933 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec15ac6edd88,46261,1733321604846 from backup master directory 2024-12-04T14:13:24,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:24,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:13:24,935 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:13:24,935 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:13:24,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:13:24,935 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:24,944 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec15ac6edd88:46261 2024-12-04T14:13:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:13:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:13:24,959 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/hbase.id with ID: bdd47805-b06c-45dd-ab18-b044fcc7e985 2024-12-04T14:13:24,974 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:24,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:24,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:24,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:13:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:13:24,992 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:13:24,993 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:13:24,993 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:13:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:13:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:13:25,409 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store 2024-12-04T14:13:25,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:13:25,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:13:25,420 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:25,420 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:13:25,420 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:25,420 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:25,421 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:13:25,421 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:25,421 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:13:25,421 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:13:25,422 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/.initializing 2024-12-04T14:13:25,422 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:25,425 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C46261%2C1733321604846, suffix=, logDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846, archiveDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/oldWALs, maxLogs=10 2024-12-04T14:13:25,426 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46261%2C1733321604846.1733321605426 2024-12-04T14:13:25,432 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 2024-12-04T14:13:25,432 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:38609:38609)] 2024-12-04T14:13:25,432 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:13:25,432 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:25,432 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,432 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:13:25,436 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:25,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:13:25,438 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:13:25,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:13:25,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:13:25,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:13:25,443 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:13:25,444 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,445 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,447 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:13:25,448 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:13:25,450 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:13:25,451 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=701619, jitterRate=-0.10784557461738586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:13:25,452 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:13:25,452 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:13:25,456 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832cfa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:13:25,457 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-04T14:13:25,457 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:13:25,457 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:13:25,457 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:13:25,457 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T14:13:25,458 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-04T14:13:25,458 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:13:25,460 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:13:25,461 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:13:25,463 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:13:25,463 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:13:25,464 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:13:25,465 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:13:25,465 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:13:25,466 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:13:25,468 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:13:25,468 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:13:25,470 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:13:25,471 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:13:25,472 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:13:25,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:13:25,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:13:25,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,475 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ec15ac6edd88,46261,1733321604846, sessionid=0x1005d6077c20000, setting cluster-up flag (Was=false) 2024-12-04T14:13:25,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,483 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:13:25,484 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:25,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,492 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:13:25,493 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:25,496 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-04T14:13:25,496 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-04T14:13:25,497 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:13:25,497 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec15ac6edd88,46261,1733321604846 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:13:25,497 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:13:25,497 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:13:25,497 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:13:25,497 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:13:25,497 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec15ac6edd88:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:13:25,498 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,498 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:13:25,498 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,498 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733321635498 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:13:25,499 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,499 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:13:25,499 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:13:25,500 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:13:25,500 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:13:25,500 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:13:25,500 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:13:25,500 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:13:25,501 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,501 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:13:25,514 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321605500,5,FailOnTimeoutGroup] 2024-12-04T14:13:25,515 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321605514,5,FailOnTimeoutGroup] 2024-12-04T14:13:25,515 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,515 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:13:25,515 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,515 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:13:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:13:25,528 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-04T14:13:25,528 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8 2024-12-04T14:13:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:13:25,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:13:25,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:25,544 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec15ac6edd88:40113 2024-12-04T14:13:25,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:13:25,545 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1008): ClusterId : bdd47805-b06c-45dd-ab18-b044fcc7e985 2024-12-04T14:13:25,545 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:13:25,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:13:25,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:25,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:13:25,548 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:13:25,549 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:13:25,549 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:13:25,549 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:25,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:13:25,551 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:13:25,551 DEBUG [RS:0;ec15ac6edd88:40113 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68336394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:13:25,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:13:25,552 DEBUG [RS:0;ec15ac6edd88:40113 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259c8fee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:13:25,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:25,552 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:13:25,552 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:13:25,552 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:13:25,552 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:25,553 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,46261,1733321604846 with isa=ec15ac6edd88/172.17.0.2:40113, startcode=1733321604895 2024-12-04T14:13:25,553 DEBUG [RS:0;ec15ac6edd88:40113 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:13:25,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/meta/1588230740 2024-12-04T14:13:25,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/meta/1588230740 2024-12-04T14:13:25,556 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:13:25,556 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41959, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:13:25,557 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46261 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,557 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46261 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:13:25,559 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8 2024-12-04T14:13:25,559 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37629 2024-12-04T14:13:25,559 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:13:25,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:13:25,561 DEBUG [RS:0;ec15ac6edd88:40113 {}] zookeeper.ZKUtil(111): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,561 WARN [RS:0;ec15ac6edd88:40113 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:13:25,561 INFO [RS:0;ec15ac6edd88:40113 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:13:25,561 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:13:25,562 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,562 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,40113,1733321604895] 2024-12-04T14:13:25,562 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873353, jitterRate=0.11052648723125458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:13:25,563 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:13:25,563 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:13:25,563 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:13:25,563 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:13:25,563 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:13:25,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:13:25,566 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:13:25,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:13:25,567 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:13:25,567 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-04T14:13:25,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:13:25,569 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:13:25,569 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:13:25,571 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:13:25,572 INFO [RS:0;ec15ac6edd88:40113 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:13:25,572 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,572 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:13:25,572 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:13:25,573 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:13:25,574 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:13:25,574 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,575 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,575 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,575 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,575 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:25,575 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:13:25,575 DEBUG [RS:0;ec15ac6edd88:40113 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:13:25,581 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,581 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,582 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,582 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,582 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40113,1733321604895-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:13:25,598 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:13:25,598 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40113,1733321604895-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:25,614 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.Replication(204): ec15ac6edd88,40113,1733321604895 started 2024-12-04T14:13:25,615 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,40113,1733321604895, RpcServer on ec15ac6edd88/172.17.0.2:40113, sessionid=0x1005d6077c20001 2024-12-04T14:13:25,615 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:13:25,615 DEBUG [RS:0;ec15ac6edd88:40113 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,615 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,40113,1733321604895' 2024-12-04T14:13:25,615 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:13:25,616 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:13:25,616 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:13:25,616 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:13:25,616 DEBUG [RS:0;ec15ac6edd88:40113 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,616 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,40113,1733321604895' 2024-12-04T14:13:25,616 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:13:25,617 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:13:25,617 DEBUG [RS:0;ec15ac6edd88:40113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:13:25,617 INFO [RS:0;ec15ac6edd88:40113 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:13:25,617 INFO [RS:0;ec15ac6edd88:40113 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:13:25,720 INFO [RS:0;ec15ac6edd88:40113 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C40113%2C1733321604895, suffix=, logDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895, archiveDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs, maxLogs=32 2024-12-04T14:13:25,722 INFO [RS:0;ec15ac6edd88:40113 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.1733321605722 2024-12-04T14:13:25,724 WARN [ec15ac6edd88:46261 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-04T14:13:25,730 INFO [RS:0;ec15ac6edd88:40113 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 2024-12-04T14:13:25,730 DEBUG [RS:0;ec15ac6edd88:40113 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:38609:38609)] 2024-12-04T14:13:25,974 DEBUG [ec15ac6edd88:46261 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:13:25,974 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:25,976 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,40113,1733321604895, state=OPENING 2024-12-04T14:13:25,978 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:13:25,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:25,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ec15ac6edd88,40113,1733321604895}] 2024-12-04T14:13:25,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:13:25,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:13:26,132 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:26,132 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:13:26,135 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36574, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:13:26,140 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-04T14:13:26,140 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:13:26,142 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C40113%2C1733321604895.meta, suffix=.meta, logDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895, archiveDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs, maxLogs=32 2024-12-04T14:13:26,144 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta 2024-12-04T14:13:26,153 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta 2024-12-04T14:13:26,153 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38609:38609),(127.0.0.1/127.0.0.1:36305:36305)] 2024-12-04T14:13:26,153 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:13:26,154 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:13:26,154 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:13:26,154 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:13:26,154 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:13:26,154 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:26,155 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-04T14:13:26,155 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-04T14:13:26,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:13:26,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:13:26,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:26,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:26,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:13:26,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:13:26,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:26,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:26,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:13:26,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:13:26,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:26,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:13:26,163 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/meta/1588230740 2024-12-04T14:13:26,164 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/meta/1588230740 2024-12-04T14:13:26,166 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:13:26,168 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:13:26,169 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855600, jitterRate=0.08795216679573059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:13:26,170 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:13:26,171 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733321606132 2024-12-04T14:13:26,173 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:13:26,174 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-04T14:13:26,174 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:26,175 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,40113,1733321604895, state=OPEN 2024-12-04T14:13:26,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:13:26,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:13:26,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:13:26,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:13:26,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:13:26,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ec15ac6edd88,40113,1733321604895 in 200 msec 2024-12-04T14:13:26,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:13:26,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 615 msec 2024-12-04T14:13:26,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 692 msec 2024-12-04T14:13:26,189 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733321606189, completionTime=-1 2024-12-04T14:13:26,189 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:13:26,189 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-04T14:13:26,190 DEBUG [hconnection-0x6ff0e812-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:13:26,192 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36584, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:13:26,193 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-04T14:13:26,193 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733321666193 2024-12-04T14:13:26,193 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733321726193 2024-12-04T14:13:26,193 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 4 msec 2024-12-04T14:13:26,199 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46261,1733321604846-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,199 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46261,1733321604846-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,199 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46261,1733321604846-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,199 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec15ac6edd88:46261, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,199 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,200 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-04T14:13:26,200 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:13:26,201 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-04T14:13:26,201 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-04T14:13:26,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:13:26,203 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:26,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:13:26,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:13:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:13:26,215 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3698606d3ffdfdf37d93cf38a28c3e9f, NAME => 'hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8 2024-12-04T14:13:26,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:13:26,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:13:26,224 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:26,224 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 3698606d3ffdfdf37d93cf38a28c3e9f, disabling compactions & flushes 2024-12-04T14:13:26,224 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,224 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,224 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. after waiting 0 ms 2024-12-04T14:13:26,224 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,224 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,224 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3698606d3ffdfdf37d93cf38a28c3e9f: 2024-12-04T14:13:26,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:13:26,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733321606226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321606226"}]},"ts":"1733321606226"} 2024-12-04T14:13:26,229 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:13:26,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:13:26,231 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321606231"}]},"ts":"1733321606231"} 2024-12-04T14:13:26,233 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-04T14:13:26,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=3698606d3ffdfdf37d93cf38a28c3e9f, ASSIGN}] 2024-12-04T14:13:26,238 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=3698606d3ffdfdf37d93cf38a28c3e9f, ASSIGN 2024-12-04T14:13:26,240 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=3698606d3ffdfdf37d93cf38a28c3e9f, ASSIGN; state=OFFLINE, location=ec15ac6edd88,40113,1733321604895; forceNewPlan=false, retain=false 2024-12-04T14:13:26,390 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=3698606d3ffdfdf37d93cf38a28c3e9f, regionState=OPENING, regionLocation=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:26,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 3698606d3ffdfdf37d93cf38a28c3e9f, server=ec15ac6edd88,40113,1733321604895}] 2024-12-04T14:13:26,546 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:26,551 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,551 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 3698606d3ffdfdf37d93cf38a28c3e9f, NAME => 'hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:13:26,551 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,552 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:26,552 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,552 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,553 INFO [StoreOpener-3698606d3ffdfdf37d93cf38a28c3e9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,555 INFO [StoreOpener-3698606d3ffdfdf37d93cf38a28c3e9f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3698606d3ffdfdf37d93cf38a28c3e9f columnFamilyName info 2024-12-04T14:13:26,555 DEBUG [StoreOpener-3698606d3ffdfdf37d93cf38a28c3e9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:26,556 INFO [StoreOpener-3698606d3ffdfdf37d93cf38a28c3e9f-1 {}] regionserver.HStore(327): Store=3698606d3ffdfdf37d93cf38a28c3e9f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:13:26,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/namespace/3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/namespace/3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:13:26,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/hbase/namespace/3698606d3ffdfdf37d93cf38a28c3e9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:13:26,562 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 3698606d3ffdfdf37d93cf38a28c3e9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=742873, jitterRate=-0.055388376116752625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:13:26,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 3698606d3ffdfdf37d93cf38a28c3e9f: 2024-12-04T14:13:26,564 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f., pid=6, masterSystemTime=1733321606546 2024-12-04T14:13:26,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,567 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:13:26,568 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=3698606d3ffdfdf37d93cf38a28c3e9f, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:26,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:13:26,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 3698606d3ffdfdf37d93cf38a28c3e9f, server=ec15ac6edd88,40113,1733321604895 in 177 msec 2024-12-04T14:13:26,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:13:26,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=3698606d3ffdfdf37d93cf38a28c3e9f, ASSIGN in 336 msec 2024-12-04T14:13:26,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:13:26,576 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321606576"}]},"ts":"1733321606576"} 2024-12-04T14:13:26,578 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-04T14:13:26,581 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:13:26,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 381 msec 2024-12-04T14:13:26,602 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-04T14:13:26,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:13:26,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:26,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:13:26,610 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-04T14:13:26,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:13:26,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 14 msec 2024-12-04T14:13:26,633 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-04T14:13:26,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:13:26,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 13 msec 2024-12-04T14:13:26,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-04T14:13:26,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.726sec 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:13:26,661 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46261,1733321604846-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:13:26,662 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46261,1733321604846-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:13:26,664 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:13:26,664 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:13:26,664 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46261,1733321604846-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,727 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7faa2a92 to 127.0.0.1:52543 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6fb20afc 2024-12-04T14:13:26,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67463d7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:13:26,733 DEBUG [hconnection-0x4a43ea9a-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:13:26,735 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36600, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:13:26,737 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ec15ac6edd88,46261,1733321604846 2024-12-04T14:13:26,738 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:26,740 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-04T14:13:26,757 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:13:26,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:26,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:26,757 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:13:26,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:13:26,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:13:26,757 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:13:26,758 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:13:26,758 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:34779 2024-12-04T14:13:26,759 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:13:26,759 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:13:26,760 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:26,762 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:13:26,765 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:34779 connecting to ZooKeeper ensemble=127.0.0.1:52543 2024-12-04T14:13:26,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347790x0, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:13:26,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:13:26,769 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34779-0x1005d6077c20003 connected 2024-12-04T14:13:26,770 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-04T14:13:26,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:13:26,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34779 2024-12-04T14:13:26,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34779 2024-12-04T14:13:26,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34779 2024-12-04T14:13:26,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34779 2024-12-04T14:13:26,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34779 2024-12-04T14:13:26,781 DEBUG [pool-282-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-04T14:13:26,793 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ec15ac6edd88:34779 2024-12-04T14:13:26,794 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1008): ClusterId : bdd47805-b06c-45dd-ab18-b044fcc7e985 2024-12-04T14:13:26,794 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:13:26,796 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:13:26,797 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:13:26,799 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:13:26,799 DEBUG [RS:1;ec15ac6edd88:34779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1868ee56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:13:26,800 DEBUG [RS:1;ec15ac6edd88:34779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b1b2d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:13:26,800 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:13:26,800 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:13:26,800 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:13:26,801 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,46261,1733321604846 with isa=ec15ac6edd88/172.17.0.2:34779, startcode=1733321606757 2024-12-04T14:13:26,801 DEBUG [RS:1;ec15ac6edd88:34779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:13:26,804 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36693, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:13:26,804 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46261 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,34779,1733321606757 2024-12-04T14:13:26,804 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46261 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,34779,1733321606757 2024-12-04T14:13:26,806 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8 2024-12-04T14:13:26,806 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37629 2024-12-04T14:13:26,806 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:13:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:13:26,809 DEBUG [RS:1;ec15ac6edd88:34779 {}] zookeeper.ZKUtil(111): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,34779,1733321606757 2024-12-04T14:13:26,809 WARN [RS:1;ec15ac6edd88:34779 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:13:26,809 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,34779,1733321606757] 2024-12-04T14:13:26,809 INFO [RS:1;ec15ac6edd88:34779 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:13:26,809 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,34779,1733321606757 2024-12-04T14:13:26,813 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:13:26,813 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:13:26,816 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:13:26,816 INFO [RS:1;ec15ac6edd88:34779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:13:26,816 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,816 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:13:26,817 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:13:26,818 DEBUG [RS:1;ec15ac6edd88:34779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:13:26,819 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,819 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,819 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,819 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,819 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,34779,1733321606757-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:13:26,835 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:13:26,835 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,34779,1733321606757-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:13:26,850 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.Replication(204): ec15ac6edd88,34779,1733321606757 started 2024-12-04T14:13:26,851 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,34779,1733321606757, RpcServer on ec15ac6edd88/172.17.0.2:34779, sessionid=0x1005d6077c20003 2024-12-04T14:13:26,851 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:13:26,851 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3355): Started new server=Thread[RS:1;ec15ac6edd88:34779,5,FailOnTimeoutGroup] 2024-12-04T14:13:26,851 DEBUG [RS:1;ec15ac6edd88:34779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,34779,1733321606757 2024-12-04T14:13:26,851 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,34779,1733321606757' 2024-12-04T14:13:26,851 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:13:26,851 INFO [Time-limited test {}] wal.TestLogRolling(191): Replication=2 2024-12-04T14:13:26,851 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:13:26,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:13:26,852 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:13:26,852 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:13:26,852 DEBUG [RS:1;ec15ac6edd88:34779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,34779,1733321606757 2024-12-04T14:13:26,852 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,34779,1733321606757' 2024-12-04T14:13:26,852 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:13:26,852 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:13:26,853 DEBUG [RS:1;ec15ac6edd88:34779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:13:26,853 INFO [RS:1;ec15ac6edd88:34779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:13:26,853 INFO [RS:1;ec15ac6edd88:34779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:13:26,854 INFO [RS-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43658, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:13:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T14:13:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T14:13:26,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:13:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T14:13:26,859 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:13:26,859 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:26,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 9 2024-12-04T14:13:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:13:26,861 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:13:26,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741837_1013 (size=393) 2024-12-04T14:13:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741837_1013 (size=393) 2024-12-04T14:13:26,871 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a57a367736daf97658ba8ac270bf7f02, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8 2024-12-04T14:13:26,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46075 is added to blk_1073741838_1014 (size=76) 2024-12-04T14:13:26,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35023 is added to blk_1073741838_1014 (size=76) 2024-12-04T14:13:26,879 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:26,879 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1681): Closing a57a367736daf97658ba8ac270bf7f02, disabling compactions & flushes 2024-12-04T14:13:26,879 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:26,880 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:26,880 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. after waiting 0 ms 2024-12-04T14:13:26,880 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:26,880 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:26,880 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1635): Region close journal for a57a367736daf97658ba8ac270bf7f02: 2024-12-04T14:13:26,881 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:13:26,882 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733321606881"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321606881"}]},"ts":"1733321606881"} 2024-12-04T14:13:26,884 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:13:26,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:13:26,885 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321606885"}]},"ts":"1733321606885"} 2024-12-04T14:13:26,887 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-12-04T14:13:26,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a57a367736daf97658ba8ac270bf7f02, ASSIGN}] 2024-12-04T14:13:26,892 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a57a367736daf97658ba8ac270bf7f02, ASSIGN 2024-12-04T14:13:26,893 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a57a367736daf97658ba8ac270bf7f02, ASSIGN; state=OFFLINE, location=ec15ac6edd88,40113,1733321604895; forceNewPlan=false, retain=false 2024-12-04T14:13:26,955 INFO [RS:1;ec15ac6edd88:34779 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C34779%2C1733321606757, suffix=, logDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,34779,1733321606757, archiveDir=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs, maxLogs=32 2024-12-04T14:13:26,957 INFO [RS:1;ec15ac6edd88:34779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C34779%2C1733321606757.1733321606956 2024-12-04T14:13:26,964 INFO [RS:1;ec15ac6edd88:34779 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,34779,1733321606757/ec15ac6edd88%2C34779%2C1733321606757.1733321606956 2024-12-04T14:13:26,964 DEBUG [RS:1;ec15ac6edd88:34779 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38609:38609),(127.0.0.1/127.0.0.1:36305:36305)] 2024-12-04T14:13:27,045 INFO [ec15ac6edd88:46261 {}] balancer.BaseLoadBalancer(546): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T14:13:27,045 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=a57a367736daf97658ba8ac270bf7f02, regionState=OPENING, regionLocation=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:27,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure a57a367736daf97658ba8ac270bf7f02, server=ec15ac6edd88,40113,1733321604895}] 2024-12-04T14:13:27,201 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:27,205 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:27,206 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => a57a367736daf97658ba8ac270bf7f02, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:13:27,206 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,206 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:13:27,207 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,207 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,208 INFO [StoreOpener-a57a367736daf97658ba8ac270bf7f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,210 INFO [StoreOpener-a57a367736daf97658ba8ac270bf7f02-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a57a367736daf97658ba8ac270bf7f02 columnFamilyName info 2024-12-04T14:13:27,210 DEBUG [StoreOpener-a57a367736daf97658ba8ac270bf7f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:13:27,211 INFO [StoreOpener-a57a367736daf97658ba8ac270bf7f02-1 {}] regionserver.HStore(327): Store=a57a367736daf97658ba8ac270bf7f02/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:13:27,212 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,212 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,214 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:27,217 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:13:27,218 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened a57a367736daf97658ba8ac270bf7f02; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844784, jitterRate=0.07419854402542114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:13:27,219 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for a57a367736daf97658ba8ac270bf7f02: 2024-12-04T14:13:27,220 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02., pid=11, masterSystemTime=1733321607201 2024-12-04T14:13:27,222 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:27,222 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:27,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=a57a367736daf97658ba8ac270bf7f02, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,40113,1733321604895 2024-12-04T14:13:27,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T14:13:27,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure a57a367736daf97658ba8ac270bf7f02, server=ec15ac6edd88,40113,1733321604895 in 177 msec 2024-12-04T14:13:27,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:13:27,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a57a367736daf97658ba8ac270bf7f02, ASSIGN in 337 msec 2024-12-04T14:13:27,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:13:27,232 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321607232"}]},"ts":"1733321607232"} 2024-12-04T14:13:27,234 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-12-04T14:13:27,237 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:13:27,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 381 msec 2024-12-04T14:13:28,525 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:28,536 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:29,053 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:13:29,056 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:29,083 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:31,569 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T14:13:31,570 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-04T14:13:31,570 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-12-04T14:13:33,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T14:13:33,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-12-04T14:13:33,241 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-04T14:13:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46261 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:13:36,862 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath, procId: 9 completed 2024-12-04T14:13:36,865 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T14:13:36,865 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:13:36,879 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:36,882 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:36,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:36,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:36,883 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:13:36,884 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a5c0625{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:36,885 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26ef4839{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:37,003 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26a64fea{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-40065-hadoop-hdfs-3_4_1-tests_jar-_-any-13748518630365429972/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:37,003 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26fbceb2{HTTP/1.1, (http/1.1)}{localhost:40065} 2024-12-04T14:13:37,004 INFO [Time-limited test {}] server.Server(415): Started @129474ms 2024-12-04T14:13:37,005 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:13:37,045 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:37,049 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:37,050 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:37,050 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:37,050 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:13:37,050 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@467ef3f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:37,051 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@614101fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:37,096 WARN [Thread-632 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data5/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:37,096 WARN [Thread-633 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data6/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:37,122 WARN [Thread-612 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:13:37,125 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74f3988b9982e6bb with lease ID 0xb59c7031934b02d7: Processing first storage report for DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa from datanode DatanodeRegistration(127.0.0.1:35835, datanodeUuid=06bd2e23-70c1-4ed3-8188-7cf9e793af30, infoPort=35695, infoSecurePort=0, ipcPort=38333, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74f3988b9982e6bb with lease ID 0xb59c7031934b02d7: from storage DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa node DatanodeRegistration(127.0.0.1:35835, datanodeUuid=06bd2e23-70c1-4ed3-8188-7cf9e793af30, infoPort=35695, infoSecurePort=0, ipcPort=38333, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74f3988b9982e6bb with lease ID 0xb59c7031934b02d7: Processing first storage report for DS-821008ec-3437-43e0-9b42-5f41a3ddc98f from datanode DatanodeRegistration(127.0.0.1:35835, datanodeUuid=06bd2e23-70c1-4ed3-8188-7cf9e793af30, infoPort=35695, infoSecurePort=0, ipcPort=38333, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74f3988b9982e6bb with lease ID 0xb59c7031934b02d7: from storage DS-821008ec-3437-43e0-9b42-5f41a3ddc98f node DatanodeRegistration(127.0.0.1:35835, datanodeUuid=06bd2e23-70c1-4ed3-8188-7cf9e793af30, infoPort=35695, infoSecurePort=0, ipcPort=38333, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:37,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@63b5442{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-39937-hadoop-hdfs-3_4_1-tests_jar-_-any-2829551536503985221/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:37,170 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54828837{HTTP/1.1, (http/1.1)}{localhost:39937} 2024-12-04T14:13:37,170 INFO [Time-limited test {}] server.Server(415): Started @129640ms 2024-12-04T14:13:37,171 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:13:37,221 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:37,225 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:37,229 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:37,229 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:37,229 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:13:37,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e253584{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:37,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cf78afc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:37,267 WARN [Thread-667 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:37,267 WARN [Thread-668 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:37,287 WARN [Thread-647 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:13:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda3273a3d429a12 with lease ID 0xb59c7031934b02d8: Processing first storage report for DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353 from datanode DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda3273a3d429a12 with lease ID 0xb59c7031934b02d8: from storage DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353 node DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda3273a3d429a12 with lease ID 0xb59c7031934b02d8: Processing first storage report for DS-01728697-f01d-4625-8c64-917a148c31e8 from datanode DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda3273a3d429a12 with lease ID 0xb59c7031934b02d8: from storage DS-01728697-f01d-4625-8c64-917a148c31e8 node DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:37,351 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@184410de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-35363-hadoop-hdfs-3_4_1-tests_jar-_-any-7926731702285162995/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:37,352 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a63e55b{HTTP/1.1, (http/1.1)}{localhost:35363} 2024-12-04T14:13:37,352 INFO [Time-limited test {}] server.Server(415): Started @129822ms 2024-12-04T14:13:37,354 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:13:37,445 WARN [Thread-693 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data9/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:37,445 WARN [Thread-694 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data10/current/BP-2091982244-172.17.0.2-1733321604097/current, will proceed with Du for space computation calculation, 2024-12-04T14:13:37,463 WARN [Thread-682 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:13:37,466 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x72370bcf4c5e2263 with lease ID 0xb59c7031934b02d9: Processing first storage report for DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31 from datanode DatanodeRegistration(127.0.0.1:43781, datanodeUuid=e7942614-1773-4509-b62b-054ff80a541f, infoPort=45861, infoSecurePort=0, ipcPort=45889, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:37,466 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x72370bcf4c5e2263 with lease ID 0xb59c7031934b02d9: from storage DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31 node DatanodeRegistration(127.0.0.1:43781, datanodeUuid=e7942614-1773-4509-b62b-054ff80a541f, infoPort=45861, infoSecurePort=0, ipcPort=45889, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x72370bcf4c5e2263 with lease ID 0xb59c7031934b02d9: Processing first storage report for DS-d1e48423-bb0e-427c-aae8-a233fe812b83 from datanode DatanodeRegistration(127.0.0.1:43781, datanodeUuid=e7942614-1773-4509-b62b-054ff80a541f, infoPort=45861, infoSecurePort=0, ipcPort=45889, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097) 2024-12-04T14:13:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x72370bcf4c5e2263 with lease ID 0xb59c7031934b02d9: from storage DS-d1e48423-bb0e-427c-aae8-a233fe812b83 node DatanodeRegistration(127.0.0.1:43781, datanodeUuid=e7942614-1773-4509-b62b-054ff80a541f, infoPort=45861, infoSecurePort=0, ipcPort=45889, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:37,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3d3898a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:37,482 WARN [ResponseProcessor for block BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,483 WARN [ResponseProcessor for block BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,488 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta block BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:37,483 WARN [ResponseProcessor for block BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,489 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK], DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:37,489 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK], DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:37,482 WARN [ResponseProcessor for block BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,490 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,34779,1733321606757/ec15ac6edd88%2C34779%2C1733321606757.1733321606956 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:37,490 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7e8dad2b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:37,490 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:37,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f686eed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:37,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:51464 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51464 dst: /127.0.0.1:35023 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e043b3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:37,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:33424 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46075:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33424 dst: /127.0.0.1:46075 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,491 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_66332805_22 at /127.0.0.1:33468 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:46075:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33468 dst: /127.0.0.1:46075 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,492 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_66332805_22 at /127.0.0.1:51482 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:35023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51482 dst: /127.0.0.1:35023 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,489 WARN [PacketResponder: BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35023] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,493 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:51414 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51414 dst: /127.0.0.1:35023 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 49745 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,494 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:13:37,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:13:37,494 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2091982244-172.17.0.2-1733321604097 (Datanode Uuid e21c3f00-8922-4a6f-b06a-1ef7593c7291) service to localhost/127.0.0.1:37629 2024-12-04T14:13:37,494 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:13:37,494 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:33408 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46075:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33408 dst: /127.0.0.1:46075 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,494 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:51444 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51444 dst: /127.0.0.1:35023 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 49726 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data3/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:37,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data4/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:37,496 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:13:37,498 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,498 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta block BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,493 WARN [PacketResponder: BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35023] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:33378 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46075:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33378 dst: /127.0.0.1:46075 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,500 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@4f7abbc3 {}] datanode.DataXceiver(331): 127.0.0.1:46075:DataXceiver error processing unknown operation src: /127.0.0.1:43622 dst: /127.0.0.1:46075 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:37,500 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,501 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,34779,1733321606757/ec15ac6edd88%2C34779%2C1733321606757.1733321606956 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741839_1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1015 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,511 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a43de3a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:37,511 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@183f70b1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:37,512 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:37,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64f9fde3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:37,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ad3a453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:37,513 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:13:37,513 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:13:37,513 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:13:37,513 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2091982244-172.17.0.2-1733321604097 (Datanode Uuid c78f2261-d155-4e80-b7c4-fb67ad306bcc) service to localhost/127.0.0.1:37629 2024-12-04T14:13:37,514 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data1/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:37,514 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data2/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:37,514 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:13:37,519 WARN [RS:0;ec15ac6edd88:40113.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=4, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,520 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C40113%2C1733321604895:(num 1733321605722) roll requested 2024-12-04T14:13:37,520 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.1733321617520 2024-12-04T14:13:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40113 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40113 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36600 deadline: 1733321627519, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL 2024-12-04T14:13:37,524 WARN [Thread-705 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1020 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,524 WARN [Thread-705 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:37,524 WARN [Thread-705 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741840_1020 2024-12-04T14:13:37,527 WARN [Thread-705 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] 2024-12-04T14:13:37,534 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL 2024-12-04T14:13:37,535 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 with entries=4, filesize=959 B; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 2024-12-04T14:13:37,535 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35695:35695),(127.0.0.1/127.0.0.1:45861:45861)] 2024-12-04T14:13:37,535 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:37,535 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,535 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:37,536 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T14:13:37,536 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T14:13:37,536 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 2024-12-04T14:13:37,539 WARN [IPC Server handler 1 on default port 37629 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741833_1009 2024-12-04T14:13:37,541 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 after 5ms 2024-12-04T14:13:38,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:13:38,745 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:38,764 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:38,767 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:38,768 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:13:41,542 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 after 4006ms 2024-12-04T14:13:49,608 INFO [Time-limited test {}] wal.TestLogRolling(243): log.getCurrentFileName(): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 2024-12-04T14:13:49,608 WARN [ResponseProcessor for block BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:49,609 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]) is bad. 2024-12-04T14:13:49,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:56556 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35835:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56556 dst: /127.0.0.1:35835 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:49,610 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60000 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1021] {}] datanode.DataXceiver(331): 127.0.0.1:43781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60000 dst: /127.0.0.1:43781 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:49,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26a64fea{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:49,611 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26fbceb2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:49,611 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:49,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26ef4839{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:49,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a5c0625{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:49,614 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:13:49,614 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:13:49,614 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:13:49,614 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2091982244-172.17.0.2-1733321604097 (Datanode Uuid 06bd2e23-70c1-4ed3-8188-7cf9e793af30) service to localhost/127.0.0.1:37629 2024-12-04T14:13:49,615 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data5/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:49,615 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data6/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:49,616 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:13:49,618 WARN [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]] 2024-12-04T14:13:49,619 WARN [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]] 2024-12-04T14:13:49,619 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C40113%2C1733321604895:(num 1733321617520) roll requested 2024-12-04T14:13:49,619 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.1733321629619 2024-12-04T14:13:49,623 WARN [Thread-715 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35023 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:49,623 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:34240 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741842_1024] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8]'}, localName='127.0.0.1:41727', datanodeUuid='1bae229d-9a97-48a7-a967-3872b67416ad', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741842_1024 to mirror 127.0.0.1:35023 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:49,624 WARN [Thread-715 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK], DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:49,624 WARN [Thread-715 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741842_1024 2024-12-04T14:13:49,624 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:34240 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741842_1024] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T14:13:49,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:34240 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:41727:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34240 dst: /127.0.0.1:41727 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:49,624 WARN [Thread-715 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] 2024-12-04T14:13:49,625 WARN [Thread-715 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:49,626 WARN [Thread-715 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]) is bad. 2024-12-04T14:13:49,626 WARN [Thread-715 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741843_1025 2024-12-04T14:13:49,626 WARN [Thread-715 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK] 2024-12-04T14:13:49,628 WARN [Thread-715 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1026 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35835 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:49,628 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40384 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741844_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data10]'}, localName='127.0.0.1:43781', datanodeUuid='e7942614-1773-4509-b62b-054ff80a541f', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741844_1026 to mirror 127.0.0.1:35835 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:49,629 WARN [Thread-715 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]) is bad. 2024-12-04T14:13:49,629 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40384 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741844_1026] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T14:13:49,629 WARN [Thread-715 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741844_1026 2024-12-04T14:13:49,629 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40384 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:43781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40384 dst: /127.0.0.1:43781 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:49,629 WARN [Thread-715 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK] 2024-12-04T14:13:49,635 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 2024-12-04T14:13:49,635 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:45861:45861)] 2024-12-04T14:13:49,635 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:49,635 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 is not closed yet, will try archiving it next time 2024-12-04T14:13:49,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43781 is added to blk_1073741841_1023 (size=2431) 2024-12-04T14:13:49,638 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:51,479 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@7303f80d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43781, datanodeUuid=e7942614-1773-4509-b62b-054ff80a541f, infoPort=45861, infoSecurePort=0, ipcPort=45889, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741841_1023 to 127.0.0.1:35835 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,622 WARN [ResponseProcessor for block BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027 java.io.IOException: Bad response ERROR for BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027 from datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,623 WARN [DataStreamer for file /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 block BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:13:53,623 WARN [PacketResponder: BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43781] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,623 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:34246 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027] {}] datanode.DataXceiver(331): 127.0.0.1:41727:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34246 dst: /127.0.0.1:41727 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40386 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1027] {}] datanode.DataXceiver(331): 127.0.0.1:43781:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40386 dst: /127.0.0.1:43781 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@184410de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:53,625 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a63e55b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:13:53,625 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:13:53,626 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cf78afc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:13:53,626 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e253584{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:13:53,628 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:13:53,628 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:13:53,628 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:13:53,628 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2091982244-172.17.0.2-1733321604097 (Datanode Uuid e7942614-1773-4509-b62b-054ff80a541f) service to localhost/127.0.0.1:37629 2024-12-04T14:13:53,628 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data9/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:53,629 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data10/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:13:53,629 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:13:53,631 WARN [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]] 2024-12-04T14:13:53,631 WARN [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]] 2024-12-04T14:13:53,631 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C40113%2C1733321604895:(num 1733321629619) roll requested 2024-12-04T14:13:53,631 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.1733321633631 2024-12-04T14:13:53,636 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35835 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,636 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60456 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8]'}, localName='127.0.0.1:41727', datanodeUuid='1bae229d-9a97-48a7-a967-3872b67416ad', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741846_1029 to mirror 127.0.0.1:35835 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,636 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK], DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]) is bad. 2024-12-04T14:13:53,636 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741846_1029 2024-12-04T14:13:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40113 {}] regionserver.HRegion(8581): Flush requested on a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:13:53,636 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60456 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T14:13:53,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a57a367736daf97658ba8ac270bf7f02 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:13:53,636 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60456 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:41727:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60456 dst: /127.0.0.1:41727 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,637 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK] 2024-12-04T14:13:53,638 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,638 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:53,639 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741847_1030 2024-12-04T14:13:53,640 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] 2024-12-04T14:13:53,641 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,642 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:13:53,642 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741848_1031 2024-12-04T14:13:53,642 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:13:53,644 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,644 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]) is bad. 2024-12-04T14:13:53,644 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741849_1032 2024-12-04T14:13:53,644 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK] 2024-12-04T14:13:53,645 WARN [IPC Server handler 3 on default port 37629 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T14:13:53,645 WARN [IPC Server handler 3 on default port 37629 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T14:13:53,645 WARN [IPC Server handler 3 on default port 37629 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T14:13:53,649 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 with entries=13, filesize=14.10 KB; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633631 2024-12-04T14:13:53,649 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959)] 2024-12-04T14:13:53,649 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:53,649 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 is not closed yet, will try archiving it next time 2024-12-04T14:13:53,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741845_1028 (size=14443) 2024-12-04T14:13:53,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/a5632ce53cea4215a09ae0cd81601901 is 1080, key is row0002/info:/1733321629617/Put/seqid=0 2024-12-04T14:13:53,660 WARN [Thread-731 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,661 WARN [Thread-731 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:53,661 WARN [Thread-731 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741851_1034 2024-12-04T14:13:53,661 WARN [Thread-731 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] 2024-12-04T14:13:53,662 WARN [Thread-731 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,663 WARN [Thread-731 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]) is bad. 2024-12-04T14:13:53,663 WARN [Thread-731 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741852_1035 2024-12-04T14:13:53,663 WARN [Thread-731 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK] 2024-12-04T14:13:53,665 WARN [Thread-731 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46075 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,665 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60486 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8]'}, localName='127.0.0.1:41727', datanodeUuid='1bae229d-9a97-48a7-a967-3872b67416ad', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741853_1036 to mirror 127.0.0.1:46075 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,665 WARN [Thread-731 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK], DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]) is bad. 2024-12-04T14:13:53,665 WARN [Thread-731 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741853_1036 2024-12-04T14:13:53,665 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60486 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T14:13:53,666 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60486 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:41727:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60486 dst: /127.0.0.1:41727 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,666 WARN [Thread-731 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK] 2024-12-04T14:13:53,668 WARN [Thread-731 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43781 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60500 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8]'}, localName='127.0.0.1:41727', datanodeUuid='1bae229d-9a97-48a7-a967-3872b67416ad', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741854_1037 to mirror 127.0.0.1:43781 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,668 WARN [Thread-731 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:13:53,669 WARN [Thread-731 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741854_1037 2024-12-04T14:13:53,669 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60500 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T14:13:53,669 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60500 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:41727:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60500 dst: /127.0.0.1:41727 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,669 WARN [Thread-731 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:13:53,670 WARN [IPC Server handler 2 on default port 37629 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T14:13:53,670 WARN [IPC Server handler 2 on default port 37629 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T14:13:53,670 WARN [IPC Server handler 2 on default port 37629 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T14:13:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741855_1038 (size=10347) 2024-12-04T14:13:53,850 WARN [sync.2 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]] 2024-12-04T14:13:53,850 WARN [sync.2 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]] 2024-12-04T14:13:53,850 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C40113%2C1733321604895:(num 1733321633631) roll requested 2024-12-04T14:13:53,851 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.1733321633850 2024-12-04T14:13:53,854 WARN [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,854 WARN [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]) is bad. 2024-12-04T14:13:53,854 WARN [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741856_1039 2024-12-04T14:13:53,855 WARN [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK] 2024-12-04T14:13:53,856 WARN [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,856 WARN [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:13:53,856 WARN [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741857_1040 2024-12-04T14:13:53,857 WARN [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:13:53,858 WARN [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,858 WARN [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK], DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]) is bad. 2024-12-04T14:13:53,858 WARN [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741858_1041 2024-12-04T14:13:53,859 WARN [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK] 2024-12-04T14:13:53,861 WARN [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35023 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:53,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60516 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8]'}, localName='127.0.0.1:41727', datanodeUuid='1bae229d-9a97-48a7-a967-3872b67416ad', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741859_1042 to mirror 127.0.0.1:35023 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,861 WARN [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK], DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]) is bad. 2024-12-04T14:13:53,861 WARN [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741859_1042 2024-12-04T14:13:53,861 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60516 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T14:13:53,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:60516 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:41727:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60516 dst: /127.0.0.1:41727 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:53,862 WARN [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35023,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK] 2024-12-04T14:13:53,862 WARN [IPC Server handler 1 on default port 37629 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T14:13:53,862 WARN [IPC Server handler 1 on default port 37629 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T14:13:53,862 WARN [IPC Server handler 1 on default port 37629 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T14:13:53,866 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633631 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633850 2024-12-04T14:13:53,866 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959)] 2024-12-04T14:13:53,866 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:53,866 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 is not closed yet, will try archiving it next time 2024-12-04T14:13:53,867 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633631 is not closed yet, will try archiving it next time 2024-12-04T14:13:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741850_1033 (size=1261) 2024-12-04T14:13:54,053 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:54,053 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633631 is not closed yet, will try archiving it next time 2024-12-04T14:13:54,053 WARN [sync.4 {}] wal.FSHLog(760): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-12-04T14:13:54,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/a5632ce53cea4215a09ae0cd81601901 2024-12-04T14:13:54,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/a5632ce53cea4215a09ae0cd81601901 as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/a5632ce53cea4215a09ae0cd81601901 2024-12-04T14:13:54,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/a5632ce53cea4215a09ae0cd81601901, entries=5, sequenceid=12, filesize=10.1 K 2024-12-04T14:13:54,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for a57a367736daf97658ba8ac270bf7f02 in 457ms, sequenceid=12, compaction requested=false 2024-12-04T14:13:54,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a57a367736daf97658ba8ac270bf7f02: 2024-12-04T14:13:54,264 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:13:54,268 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:13:54,269 DEBUG [Close-WAL-Writer-2 {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321605722 is not closed yet, will try archiving it next time 2024-12-04T14:13:54,270 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 to hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs/ec15ac6edd88%2C40113%2C1733321604895.1733321617520 2024-12-04T14:13:54,270 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:13:54,270 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:13:54,271 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:13:54,271 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d1706b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:13:54,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32033cd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:13:54,387 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@351d8832{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/java.io.tmpdir/jetty-localhost-38729-hadoop-hdfs-3_4_1-tests_jar-_-any-4716059886366914605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:13:54,388 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6444fc9e{HTTP/1.1, (http/1.1)}{localhost:38729} 2024-12-04T14:13:54,388 INFO [Time-limited test {}] server.Server(415): Started @146859ms 2024-12-04T14:13:54,390 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:13:54,489 WARN [Thread-760 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:13:54,498 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9bef532152d41fb6 with lease ID 0xb59c7031934b02da: from storage DS-9222f14d-c6ad-48a7-be72-9411e899a952 node DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:13:54,498 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9bef532152d41fb6 with lease ID 0xb59c7031934b02da: from storage DS-d3639750-1cab-43fc-8d86-d75999ae8b8f node DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:13:54,824 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:13:55,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73a09617[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741855_1038 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:55,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@779f1c3c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741845_1028 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:55,499 WARN [master/ec15ac6edd88:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=96, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,499 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C46261%2C1733321604846:(num 1733321605426) roll requested 2024-12-04T14:13:55,500 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46261%2C1733321604846.1733321635499 2024-12-04T14:13:55,500 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,500 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,503 WARN [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,503 WARN [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK], DatanodeInfoWithStorage[127.0.0.1:35835,DS-4fea22ae-631b-46b8-b8c7-4777ebf559aa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]) is bad. 2024-12-04T14:13:55,503 WARN [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741861_1044 2024-12-04T14:13:55,504 WARN [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK] 2024-12-04T14:13:55,506 WARN [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43781 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,506 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:46414 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data4]'}, localName='127.0.0.1:43473', datanodeUuid='e21c3f00-8922-4a6f-b06a-1ef7593c7291', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741862_1045 to mirror 127.0.0.1:43781 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:55,507 WARN [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43473,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:13:55,507 WARN [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741862_1045 2024-12-04T14:13:55,507 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:46414 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T14:13:55,507 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:46414 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:43473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46414 dst: /127.0.0.1:43473 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:55,508 WARN [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:13:55,514 WARN [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL 2024-12-04T14:13:55,514 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 with entries=93, filesize=46.04 KB; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321635499 2024-12-04T14:13:55,514 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:38877:38877)] 2024-12-04T14:13:55,514 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 is not closed yet, will try archiving it next time 2024-12-04T14:13:55,514 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,515 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:13:55,515 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 2024-12-04T14:13:55,515 WARN [IPC Server handler 3 on default port 37629 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 has not been closed. Lease recovery is in progress. RecoveryId = 1047 for block blk_1073741830_1006 2024-12-04T14:13:55,516 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 after 1ms 2024-12-04T14:13:55,552 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:13:55,553 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34276, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:13:56,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73a09617[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741850_1033 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:13:57,800 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:13:57,802 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34292, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:13:59,517 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846/ec15ac6edd88%2C46261%2C1733321604846.1733321605426 after 4002ms 2024-12-04T14:14:00,494 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@24981f0e[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741836_1012 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741838_1014 (size=76) 2024-12-04T14:14:01,494 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@24981f0e[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741828_1004 to 127.0.0.1:35835 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:01,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:14:03,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:14:03,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:14:04,494 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@24981f0e[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741825_1001 to 127.0.0.1:35835 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:04,511 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@44b7e041 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2091982244-172.17.0.2-1733321604097:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:46075,null,null]) java.net.ConnectException: Call From ec15ac6edd88/172.17.0.2 to localhost:40497 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T14:14:04,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741833_1022 (size=959) 2024-12-04T14:14:06,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741837_1013 (size=393) 2024-12-04T14:14:06,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:14:07,058 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:14:07,058 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:14:07,494 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@24981f0e[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741831_1007 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:07,494 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5f56494d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741835_1011 to 127.0.0.1:35835 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:09,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741833_1022 (size=959) 2024-12-04T14:14:12,207 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region a57a367736daf97658ba8ac270bf7f02, had cached 0 bytes from a total of 10347 2024-12-04T14:14:13,470 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.1733321653470 2024-12-04T14:14:13,474 WARN [Thread-803 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1048 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43781 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,474 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:40788 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741864_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data4]'}, localName='127.0.0.1:43473', datanodeUuid='e21c3f00-8922-4a6f-b06a-1ef7593c7291', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741864_1048 to mirror 127.0.0.1:43781 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:13,474 WARN [Thread-803 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741864_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43473,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:14:13,474 WARN [Thread-803 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741864_1048 2024-12-04T14:14:13,474 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:40788 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741864_1048] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T14:14:13,474 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1597984038_22 at /127.0.0.1:40788 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741864_1048] {}] datanode.DataXceiver(331): 127.0.0.1:43473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40788 dst: /127.0.0.1:43473 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:13,475 WARN [Thread-803 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:14:13,480 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633850 with entries=2, filesize=1.57 KB; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321653470 2024-12-04T14:14:13,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:38877:38877)] 2024-12-04T14:14:13,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633850 is not closed yet, will try archiving it next time 2024-12-04T14:14:13,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741860_1043 (size=1618) 2024-12-04T14:14:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40113 {}] regionserver.HRegion(8581): Flush requested on a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:14:13,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a57a367736daf97658ba8ac270bf7f02 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-12-04T14:14:13,483 INFO [sync.3 {}] wal.FSHLog(777): LowReplication-Roller was enabled. 2024-12-04T14:14:13,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/fa839891691b401494b1a06f92091ad9 is 1080, key is row0007/info:/1733321633637/Put/seqid=0 2024-12-04T14:14:13,494 WARN [Thread-810 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1050 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43781 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,494 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40816 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741866_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data4]'}, localName='127.0.0.1:43473', datanodeUuid='e21c3f00-8922-4a6f-b06a-1ef7593c7291', xmitsInProgress=0}:Exception transferring block BP-2091982244-172.17.0.2-1733321604097:blk_1073741866_1050 to mirror 127.0.0.1:43781 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:13,494 WARN [Thread-810 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741866_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43473,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK], DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:14:13,494 WARN [Thread-810 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741866_1050 2024-12-04T14:14:13,494 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40816 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741866_1050] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T14:14:13,494 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539073274_22 at /127.0.0.1:40816 [Receiving block BP-2091982244-172.17.0.2-1733321604097:blk_1073741866_1050] {}] datanode.DataXceiver(331): 127.0.0.1:43473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40816 dst: /127.0.0.1:43473 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:13,495 WARN [Thread-810 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:14:13,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-04T14:14:13,496 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-04T14:14:13,496 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7faa2a92 to 127.0.0.1:52543 2024-12-04T14:14:13,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:13,497 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:14:13,497 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1436068235, stopped=false 2024-12-04T14:14:13,497 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ec15ac6edd88,46261,1733321604846 2024-12-04T14:14:13,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741867_1051 (size=13583) 2024-12-04T14:14:13,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-04T14:14:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:14:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:14:13,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741867_1051 (size=13583) 2024-12-04T14:14:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:13,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:13,500 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,40113,1733321604895' ***** 2024-12-04T14:14:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:14:13,500 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:14:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:13,500 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,34779,1733321606757' ***** 2024-12-04T14:14:13,500 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:14:13,501 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:14:13,501 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:14:13,501 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:14:13,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=25 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/fa839891691b401494b1a06f92091ad9 2024-12-04T14:14:13,501 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:14:13,502 INFO [RS:1;ec15ac6edd88:34779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:14:13,502 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:14:13,502 INFO [RS:1;ec15ac6edd88:34779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:14:13,502 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,34779,1733321606757 2024-12-04T14:14:13,502 DEBUG [RS:1;ec15ac6edd88:34779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:13,502 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,34779,1733321606757; all regions closed. 2024-12-04T14:14:13,502 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,34779,1733321606757 2024-12-04T14:14:13,502 WARN [WAL-Shutdown-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,503 ERROR [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1664): Shutdown / close of WAL failed: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... 2024-12-04T14:14:13,503 DEBUG [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1665): Shutdown / close exception details: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,503 DEBUG [RS:1;ec15ac6edd88:34779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:13,503 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:14:13,503 INFO [RS:1;ec15ac6edd88:34779 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-04T14:14:13,504 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:14:13,504 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:14:13,504 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:14:13,504 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:14:13,504 INFO [RS:1;ec15ac6edd88:34779 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:34779 2024-12-04T14:14:13,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,34779,1733321606757 2024-12-04T14:14:13,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:14:13,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:14:13,509 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,34779,1733321606757] 2024-12-04T14:14:13,509 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,34779,1733321606757; numProcessing=1 2024-12-04T14:14:13,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/fa839891691b401494b1a06f92091ad9 as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/fa839891691b401494b1a06f92091ad9 2024-12-04T14:14:13,510 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,34779,1733321606757 already deleted, retry=false 2024-12-04T14:14:13,510 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,34779,1733321606757 expired; onlineServers=1 2024-12-04T14:14:13,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/fa839891691b401494b1a06f92091ad9, entries=8, sequenceid=25, filesize=13.3 K 2024-12-04T14:14:13,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~10.50 KB/10757, heapSize ~11.48 KB/11760, currentSize=9.46 KB/9684 for a57a367736daf97658ba8ac270bf7f02 in 35ms, sequenceid=25, compaction requested=false 2024-12-04T14:14:13,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a57a367736daf97658ba8ac270bf7f02: 2024-12-04T14:14:13,517 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=23.4 K, sizeToCheck=16.0 K 2024-12-04T14:14:13,517 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:14:13,517 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/fa839891691b401494b1a06f92091ad9 because midkey is the same as first or last row 2024-12-04T14:14:13,517 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(3579): Received CLOSE for a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(3579): Received CLOSE for 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,40113,1733321604895 2024-12-04T14:14:13,518 DEBUG [RS:0;ec15ac6edd88:40113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:14:13,518 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a57a367736daf97658ba8ac270bf7f02, disabling compactions & flushes 2024-12-04T14:14:13,518 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:14:13,518 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:14:13,518 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. after waiting 0 ms 2024-12-04T14:14:13,518 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:14:13,518 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-04T14:14:13,518 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a57a367736daf97658ba8ac270bf7f02 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-12-04T14:14:13,519 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1603): Online Regions={a57a367736daf97658ba8ac270bf7f02=TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02., 1588230740=hbase:meta,,1.1588230740, 3698606d3ffdfdf37d93cf38a28c3e9f=hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f.} 2024-12-04T14:14:13,519 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:14:13,519 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:14:13,519 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 3698606d3ffdfdf37d93cf38a28c3e9f, a57a367736daf97658ba8ac270bf7f02 2024-12-04T14:14:13,519 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:14:13,519 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:14:13,519 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:14:13,519 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.87 KB heapSize=5.40 KB 2024-12-04T14:14:13,519 WARN [RS_OPEN_META-regionserver/ec15ac6edd88:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,519 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C40113%2C1733321604895.meta:.meta(num 1733321606144) roll requested 2024-12-04T14:14:13,519 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:14:13,520 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40113%2C1733321604895.meta.1733321653520.meta 2024-12-04T14:14:13,520 ERROR [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server ec15ac6edd88,40113,1733321604895: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,520 ERROR [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-04T14:14:13,523 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-04T14:14:13,524 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/b3b4ef1456d74ee8a8e445eebb277bac is 1080, key is row0014/info:/1733321653483/Put/seqid=0 2024-12-04T14:14:13,524 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-04T14:14:13,524 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-04T14:14:13,524 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-04T14:14:13,524 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 251633152 }, "NonHeapMemoryUsage": { "committed": 162922496, "init": 7667712, "max": -1, "used": 161047488 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-04T14:14:13,526 WARN [Thread-820 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,526 WARN [Thread-820 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741869_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:14:13,526 WARN [Thread-820 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741869_1053 2024-12-04T14:14:13,527 WARN [Thread-820 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:14:13,527 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL 2024-12-04T14:14:13,527 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta with entries=11, filesize=3.63 KB; new WAL /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321653520.meta 2024-12-04T14:14:13,527 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:38877:38877)] 2024-12-04T14:14:13,527 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta is not closed yet, will try archiving it next time 2024-12-04T14:14:13,527 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,528 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:13,528 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta 2024-12-04T14:14:13,528 WARN [IPC Server handler 3 on default port 37629 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta has not been closed. Lease recovery is in progress. RecoveryId = 1055 for block blk_1073741834_1010 2024-12-04T14:14:13,528 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46261 {}] master.MasterRpcServices(626): ec15ac6edd88,40113,1733321604895 reported a fatal error: ***** ABORTING region server ec15ac6edd88,40113,1733321604895: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** Cause: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46075,DS-58a778b9-546d-4598-9fc0-013c034938d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) 2024-12-04T14:14:13,528 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta after 0ms 2024-12-04T14:14:13,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741870_1054 (size=14663) 2024-12-04T14:14:13,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741870_1054 (size=14663) 2024-12-04T14:14:13,532 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/b3b4ef1456d74ee8a8e445eebb277bac 2024-12-04T14:14:13,539 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/.tmp/info/b3b4ef1456d74ee8a8e445eebb277bac as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/b3b4ef1456d74ee8a8e445eebb277bac 2024-12-04T14:14:13,545 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/info/b3b4ef1456d74ee8a8e445eebb277bac, entries=9, sequenceid=37, filesize=14.3 K 2024-12-04T14:14:13,546 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for a57a367736daf97658ba8ac270bf7f02 in 28ms, sequenceid=37, compaction requested=true 2024-12-04T14:14:13,550 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a57a367736daf97658ba8ac270bf7f02/recovered.edits/40.seqid, newMaxSeqId=40, maxSeqId=1 2024-12-04T14:14:13,550 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:14:13,550 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a57a367736daf97658ba8ac270bf7f02: 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733321606856.a57a367736daf97658ba8ac270bf7f02. 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3698606d3ffdfdf37d93cf38a28c3e9f, disabling compactions & flushes 2024-12-04T14:14:13,551 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. after waiting 0 ms 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3698606d3ffdfdf37d93cf38a28c3e9f: 2024-12-04T14:14:13,551 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,584 INFO [regionserver/ec15ac6edd88:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:14:13,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:14:13,608 INFO [RS:1;ec15ac6edd88:34779 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,34779,1733321606757; zookeeper connection closed. 2024-12-04T14:14:13,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34779-0x1005d6077c20003, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:14:13,608 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3d952b28 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3d952b28 2024-12-04T14:14:13,651 INFO [regionserver/ec15ac6edd88:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T14:14:13,651 INFO [regionserver/ec15ac6edd88:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T14:14:13,719 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:14:13,719 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(3579): Received CLOSE for 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:14:13,719 DEBUG [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 3698606d3ffdfdf37d93cf38a28c3e9f 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3698606d3ffdfdf37d93cf38a28c3e9f, disabling compactions & flushes 2024-12-04T14:14:13,719 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:14:13,719 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. after waiting 0 ms 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:14:13,719 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,720 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:14:13,720 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3698606d3ffdfdf37d93cf38a28c3e9f: 2024-12-04T14:14:13,720 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740 2024-12-04T14:14:13,720 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733321606200.3698606d3ffdfdf37d93cf38a28c3e9f. 2024-12-04T14:14:13,883 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 to hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs/ec15ac6edd88%2C40113%2C1733321604895.1733321629619 2024-12-04T14:14:13,885 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633631 to hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs/ec15ac6edd88%2C40113%2C1733321604895.1733321633631 2024-12-04T14:14:13,886 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.1733321633850 to hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/oldWALs/ec15ac6edd88%2C40113%2C1733321604895.1733321633850 2024-12-04T14:14:13,919 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing 2024-12-04T14:14:13,919 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,40113,1733321604895; all regions closed. 2024-12-04T14:14:13,920 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895 2024-12-04T14:14:13,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741868_1052 (size=93) 2024-12-04T14:14:13,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741868_1052 (size=93) 2024-12-04T14:14:14,516 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@273213b9 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:46075,null,null]) java.net.ConnectException: Call From ec15ac6edd88/172.17.0.2 to localhost:40497 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T14:14:14,821 INFO [regionserver/ec15ac6edd88:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:14:15,495 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5f56494d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43473, datanodeUuid=e21c3f00-8922-4a6f-b06a-1ef7593c7291, infoPort=38877, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741830_1047 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:16,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73a09617[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41727, datanodeUuid=1bae229d-9a97-48a7-a967-3872b67416ad, infoPort=38959, infoSecurePort=0, ipcPort=39727, storageInfo=lv=-57;cid=testClusterID;nsid=967735606;c=1733321604097):Failed to transfer BP-2091982244-172.17.0.2-1733321604097:blk_1073741860_1043 to 127.0.0.1:43781 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:17,529 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta after 4001ms 2024-12-04T14:14:18,552 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:18,567 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:18,567 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:18,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:18,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:18,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:18,923 ERROR [WAL-Shutdown-0 {}] wal.FSHLog(508): We have waited 5 seconds but the close of writer(s) doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-04T14:14:18,924 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895 2024-12-04T14:14:18,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741865_1049 (size=13514) 2024-12-04T14:14:18,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741865_1049 (size=13514) 2024-12-04T14:14:18,926 DEBUG [RS:0;ec15ac6edd88:40113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:18,927 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:14:18,927 INFO [RS:0;ec15ac6edd88:40113 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T14:14:18,927 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:14:18,927 INFO [RS:0;ec15ac6edd88:40113 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40113 2024-12-04T14:14:18,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,40113,1733321604895 2024-12-04T14:14:18,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:14:18,933 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,40113,1733321604895] 2024-12-04T14:14:18,933 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,40113,1733321604895; numProcessing=2 2024-12-04T14:14:18,934 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,40113,1733321604895 already deleted, retry=false 2024-12-04T14:14:18,934 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,40113,1733321604895 expired; onlineServers=0 2024-12-04T14:14:18,934 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,46261,1733321604846' ***** 2024-12-04T14:14:18,934 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:14:18,934 DEBUG [M:0;ec15ac6edd88:46261 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c918f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:14:18,934 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,46261,1733321604846 2024-12-04T14:14:18,934 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,46261,1733321604846; all regions closed. 2024-12-04T14:14:18,935 DEBUG [M:0;ec15ac6edd88:46261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:14:18,935 DEBUG [M:0;ec15ac6edd88:46261 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:14:18,935 DEBUG [M:0;ec15ac6edd88:46261 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:14:18,935 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:14:18,935 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321605500 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321605500,5,FailOnTimeoutGroup] 2024-12-04T14:14:18,935 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321605514 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321605514,5,FailOnTimeoutGroup] 2024-12-04T14:14:18,935 INFO [M:0;ec15ac6edd88:46261 {}] hbase.ChoreService(370): Chore service for: master/ec15ac6edd88:0 had [] on shutdown 2024-12-04T14:14:18,935 DEBUG [M:0;ec15ac6edd88:46261 {}] master.HMaster(1733): Stopping service threads 2024-12-04T14:14:18,935 INFO [M:0;ec15ac6edd88:46261 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:14:18,936 INFO [M:0;ec15ac6edd88:46261 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:14:18,936 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:14:18,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:14:18,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:18,936 DEBUG [M:0;ec15ac6edd88:46261 {}] zookeeper.ZKUtil(347): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:14:18,936 WARN [M:0;ec15ac6edd88:46261 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:14:18,936 INFO [M:0;ec15ac6edd88:46261 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-04T14:14:18,936 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:14:18,936 INFO [M:0;ec15ac6edd88:46261 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:14:18,936 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:14:18,936 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:18,937 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:18,937 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:14:18,937 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:18,937 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.29 KB 2024-12-04T14:14:18,953 DEBUG [M:0;ec15ac6edd88:46261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e708315f3534440db33977d6aa41076e is 82, key is hbase:meta,,1/info:regioninfo/1733321606174/Put/seqid=0 2024-12-04T14:14:18,955 WARN [Thread-835 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:18,956 WARN [Thread-835 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741871_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:14:18,956 WARN [Thread-835 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741871_1056 2024-12-04T14:14:18,956 WARN [Thread-835 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:14:18,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741872_1057 (size=5672) 2024-12-04T14:14:18,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741872_1057 (size=5672) 2024-12-04T14:14:18,962 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e708315f3534440db33977d6aa41076e 2024-12-04T14:14:18,983 DEBUG [M:0;ec15ac6edd88:46261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1a60a35b55bf49a8a21b58b5279e2376 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733321607238/Put/seqid=0 2024-12-04T14:14:18,984 WARN [Thread-842 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1058 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:18,985 WARN [Thread-842 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741873_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:43473,DS-9222f14d-c6ad-48a7-be72-9411e899a952,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:14:18,985 WARN [Thread-842 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741873_1058 2024-12-04T14:14:18,985 WARN [Thread-842 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:14:18,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741874_1059 (size=7465) 2024-12-04T14:14:18,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741874_1059 (size=7465) 2024-12-04T14:14:18,990 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.41 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1a60a35b55bf49a8a21b58b5279e2376 2024-12-04T14:14:19,010 DEBUG [M:0;ec15ac6edd88:46261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/55d0512761974d698c07a33856381dbd is 69, key is ec15ac6edd88,34779,1733321606757/rs:state/1733321606804/Put/seqid=0 2024-12-04T14:14:19,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741875_1060 (size=5224) 2024-12-04T14:14:19,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741875_1060 (size=5224) 2024-12-04T14:14:19,016 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/55d0512761974d698c07a33856381dbd 2024-12-04T14:14:19,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:14:19,033 INFO [RS:0;ec15ac6edd88:40113 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,40113,1733321604895; zookeeper connection closed. 2024-12-04T14:14:19,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40113-0x1005d6077c20001, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:14:19,033 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@30b7ce09 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@30b7ce09 2024-12-04T14:14:19,033 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-04T14:14:19,038 DEBUG [M:0;ec15ac6edd88:46261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b862442ac58f443f8dcf17e082aa9af0 is 52, key is load_balancer_on/state:d/1733321606739/Put/seqid=0 2024-12-04T14:14:19,040 WARN [Thread-855 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:19,040 WARN [Thread-855 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2091982244-172.17.0.2-1733321604097:blk_1073741876_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK], DatanodeInfoWithStorage[127.0.0.1:41727,DS-6b4a9278-4d61-47b7-bfe7-1ce5ab428353,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK]) is bad. 2024-12-04T14:14:19,040 WARN [Thread-855 {}] hdfs.DataStreamer(1850): Abandoning BP-2091982244-172.17.0.2-1733321604097:blk_1073741876_1061 2024-12-04T14:14:19,041 WARN [Thread-855 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43781,DS-7da7e1c9-c33e-4a0d-81cd-962c93897e31,DISK] 2024-12-04T14:14:19,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741877_1062 (size=5056) 2024-12-04T14:14:19,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741877_1062 (size=5056) 2024-12-04T14:14:19,046 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b862442ac58f443f8dcf17e082aa9af0 2024-12-04T14:14:19,051 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e708315f3534440db33977d6aa41076e as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e708315f3534440db33977d6aa41076e 2024-12-04T14:14:19,056 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e708315f3534440db33977d6aa41076e, entries=8, sequenceid=97, filesize=5.5 K 2024-12-04T14:14:19,057 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1a60a35b55bf49a8a21b58b5279e2376 as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1a60a35b55bf49a8a21b58b5279e2376 2024-12-04T14:14:19,062 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1a60a35b55bf49a8a21b58b5279e2376, entries=11, sequenceid=97, filesize=7.3 K 2024-12-04T14:14:19,063 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/55d0512761974d698c07a33856381dbd as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/55d0512761974d698c07a33856381dbd 2024-12-04T14:14:19,068 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/55d0512761974d698c07a33856381dbd, entries=2, sequenceid=97, filesize=5.1 K 2024-12-04T14:14:19,069 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b862442ac58f443f8dcf17e082aa9af0 as hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b862442ac58f443f8dcf17e082aa9af0 2024-12-04T14:14:19,074 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b862442ac58f443f8dcf17e082aa9af0, entries=1, sequenceid=97, filesize=4.9 K 2024-12-04T14:14:19,075 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41039, heapSize ~49.23 KB/50408, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 138ms, sequenceid=97, compaction requested=false 2024-12-04T14:14:19,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:14:19,078 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:19,081 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:19,081 DEBUG [M:0;ec15ac6edd88:46261 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:14:19,081 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/MasterData/WALs/ec15ac6edd88,46261,1733321604846 2024-12-04T14:14:19,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41727 is added to blk_1073741863_1046 (size=757) 2024-12-04T14:14:19,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43473 is added to blk_1073741863_1046 (size=757) 2024-12-04T14:14:19,093 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:19,094 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:19,096 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:19,096 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:19,097 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:19,491 INFO [M:0;ec15ac6edd88:46261 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-04T14:14:19,491 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:14:19,491 INFO [M:0;ec15ac6edd88:46261 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46261 2024-12-04T14:14:19,493 DEBUG [M:0;ec15ac6edd88:46261 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ec15ac6edd88,46261,1733321604846 already deleted, retry=false 2024-12-04T14:14:19,530 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:19,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:14:19,595 INFO [M:0;ec15ac6edd88:46261 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,46261,1733321604846; zookeeper connection closed. 2024-12-04T14:14:19,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46261-0x1005d6077c20000, quorum=127.0.0.1:52543, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:14:19,598 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@351d8832{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:19,598 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6444fc9e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:19,598 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:19,598 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32033cd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:19,598 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d1706b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:19,600 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:14:19,600 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:14:19,600 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2091982244-172.17.0.2-1733321604097 (Datanode Uuid e21c3f00-8922-4a6f-b06a-1ef7593c7291) service to localhost/127.0.0.1:37629 2024-12-04T14:14:19,600 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:14:19,600 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@13006fff {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:46075,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:40497 , LocalHost:localPort ec15ac6edd88/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T14:14:19,600 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@13006fff {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-2091982244-172.17.0.2-1733321604097:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:43473,null,null], DatanodeInfoWithStorage[127.0.0.1:46075,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-2091982244-172.17.0.2-1733321604097 2024-12-04T14:14:19,601 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data3/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:19,601 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data4/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:19,601 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:14:19,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@63b5442{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:19,603 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54828837{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:19,603 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:19,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@614101fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:19,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@467ef3f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:19,605 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:14:19,605 WARN [BP-2091982244-172.17.0.2-1733321604097 heartbeating to localhost/127.0.0.1:37629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2091982244-172.17.0.2-1733321604097 (Datanode Uuid 1bae229d-9a97-48a7-a967-3872b67416ad) service to localhost/127.0.0.1:37629 2024-12-04T14:14:19,606 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data7/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:19,606 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/cluster_e167aec2-f81a-8b38-c5fa-9060a7cc7567/dfs/data/data8/current/BP-2091982244-172.17.0.2-1733321604097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:19,606 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:14:19,606 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:14:19,607 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:14:19,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@654852b8{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:14:19,616 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74e82574{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:19,617 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:19,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4063a7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:19,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25d9b335{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:19,624 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:14:19,653 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-04T14:14:19,660 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=85 (was 63) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37629 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-13-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37629 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37629 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-12-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37629 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$$Lambda$796/0x00007f7e30b87468.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37629 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37629 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:37629 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37629 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:37629 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=428 (was 406) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 259), ProcessCount=11 (was 11), AvailableMemoryMB=6266 (was 6634) 2024-12-04T14:14:19,668 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=85, OpenFileDescriptor=428, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=6266 2024-12-04T14:14:19,668 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:14:19,668 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.log.dir so I do NOT create it in target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e 2024-12-04T14:14:19,668 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/86f9e898-f096-7d3e-99ba-2612da15bf30/hadoop.tmp.dir so I do NOT create it in target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e 2024-12-04T14:14:19,668 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de, deleteOnExit=true 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/test.cache.data in system properties and HBase conf 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:14:19,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-04T14:14:19,670 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:14:19,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:14:19,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:14:19,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:14:19,672 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:14:19,672 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:14:19,672 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:14:19,690 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:14:19,764 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:19,770 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:19,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:19,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:19,775 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:14:19,776 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:19,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b73fa7a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:19,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c1c3841{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:19,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b9e6474{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-34987-hadoop-hdfs-3_4_1-tests_jar-_-any-7741965075477425932/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:14:19,892 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@579b5c9d{HTTP/1.1, (http/1.1)}{localhost:34987} 2024-12-04T14:14:19,892 INFO [Time-limited test {}] server.Server(415): Started @172363ms 2024-12-04T14:14:19,905 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:14:19,974 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:19,978 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:19,978 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:19,979 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:19,979 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:14:19,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55633cc8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:19,980 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72e01b7a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:20,094 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5fd37f1f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-35703-hadoop-hdfs-3_4_1-tests_jar-_-any-3981278941086364418/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:20,095 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e67e938{HTTP/1.1, (http/1.1)}{localhost:35703} 2024-12-04T14:14:20,095 INFO [Time-limited test {}] server.Server(415): Started @172565ms 2024-12-04T14:14:20,096 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:14:20,128 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:20,131 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:20,132 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:20,132 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:20,132 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:14:20,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4671d85f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:20,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1794184d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:20,193 WARN [Thread-935 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data1/current/BP-339444668-172.17.0.2-1733321659710/current, will proceed with Du for space computation calculation, 2024-12-04T14:14:20,193 WARN [Thread-936 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data2/current/BP-339444668-172.17.0.2-1733321659710/current, will proceed with Du for space computation calculation, 2024-12-04T14:14:20,211 WARN [Thread-914 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:14:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9bf7447e88f9e1ae with lease ID 0xb0988b48613d2a94: Processing first storage report for DS-ff569953-6068-402f-bedf-7a0934dabfa9 from datanode DatanodeRegistration(127.0.0.1:35591, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=36305, infoSecurePort=0, ipcPort=39429, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710) 2024-12-04T14:14:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9bf7447e88f9e1ae with lease ID 0xb0988b48613d2a94: from storage DS-ff569953-6068-402f-bedf-7a0934dabfa9 node DatanodeRegistration(127.0.0.1:35591, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=36305, infoSecurePort=0, ipcPort=39429, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9bf7447e88f9e1ae with lease ID 0xb0988b48613d2a94: Processing first storage report for DS-77f15865-3968-43d7-a40e-aa5beceb926c from datanode DatanodeRegistration(127.0.0.1:35591, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=36305, infoSecurePort=0, ipcPort=39429, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710) 2024-12-04T14:14:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9bf7447e88f9e1ae with lease ID 0xb0988b48613d2a94: from storage DS-77f15865-3968-43d7-a40e-aa5beceb926c node DatanodeRegistration(127.0.0.1:35591, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=36305, infoSecurePort=0, ipcPort=39429, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:20,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64139653{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-43801-hadoop-hdfs-3_4_1-tests_jar-_-any-1488100928085540178/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:20,260 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3800cd12{HTTP/1.1, (http/1.1)}{localhost:43801} 2024-12-04T14:14:20,260 INFO [Time-limited test {}] server.Server(415): Started @172730ms 2024-12-04T14:14:20,261 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:14:20,352 WARN [Thread-961 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data3/current/BP-339444668-172.17.0.2-1733321659710/current, will proceed with Du for space computation calculation, 2024-12-04T14:14:20,352 WARN [Thread-962 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data4/current/BP-339444668-172.17.0.2-1733321659710/current, will proceed with Du for space computation calculation, 2024-12-04T14:14:20,375 WARN [Thread-950 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:14:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e7ec91e7cc17779 with lease ID 0xb0988b48613d2a95: Processing first storage report for DS-719143b2-faf0-4126-93d8-fd7ea76d9420 from datanode DatanodeRegistration(127.0.0.1:37939, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=35407, infoSecurePort=0, ipcPort=36703, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710) 2024-12-04T14:14:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e7ec91e7cc17779 with lease ID 0xb0988b48613d2a95: from storage DS-719143b2-faf0-4126-93d8-fd7ea76d9420 node DatanodeRegistration(127.0.0.1:37939, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=35407, infoSecurePort=0, ipcPort=36703, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e7ec91e7cc17779 with lease ID 0xb0988b48613d2a95: Processing first storage report for DS-127d24c8-5c5a-446c-899d-09b0919f5f2a from datanode DatanodeRegistration(127.0.0.1:37939, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=35407, infoSecurePort=0, ipcPort=36703, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710) 2024-12-04T14:14:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e7ec91e7cc17779 with lease ID 0xb0988b48613d2a95: from storage DS-127d24c8-5c5a-446c-899d-09b0919f5f2a node DatanodeRegistration(127.0.0.1:37939, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=35407, infoSecurePort=0, ipcPort=36703, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:20,385 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e 2024-12-04T14:14:20,388 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/zookeeper_0, clientPort=59796, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:14:20,388 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=59796 2024-12-04T14:14:20,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:14:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:14:20,400 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4 with version=8 2024-12-04T14:14:20,400 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase-staging 2024-12-04T14:14:20,402 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:14:20,403 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:14:20,404 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45395 2024-12-04T14:14:20,404 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,408 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:45395 connecting to ZooKeeper ensemble=127.0.0.1:59796 2024-12-04T14:14:20,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453950x0, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:14:20,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45395-0x1005d6150c90000 connected 2024-12-04T14:14:20,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:14:20,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:14:20,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:14:20,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45395 2024-12-04T14:14:20,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45395 2024-12-04T14:14:20,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45395 2024-12-04T14:14:20,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45395 2024-12-04T14:14:20,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45395 2024-12-04T14:14:20,432 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4, hbase.cluster.distributed=false 2024-12-04T14:14:20,448 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:14:20,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:14:20,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:14:20,448 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:14:20,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:14:20,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:14:20,448 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:14:20,449 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:14:20,449 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42781 2024-12-04T14:14:20,449 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:14:20,450 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:14:20,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,453 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,455 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:42781 connecting to ZooKeeper ensemble=127.0.0.1:59796 2024-12-04T14:14:20,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427810x0, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:14:20,458 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42781-0x1005d6150c90001 connected 2024-12-04T14:14:20,458 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:14:20,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:14:20,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:14:20,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42781 2024-12-04T14:14:20,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42781 2024-12-04T14:14:20,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42781 2024-12-04T14:14:20,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42781 2024-12-04T14:14:20,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42781 2024-12-04T14:14:20,461 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:14:20,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:14:20,463 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:14:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:14:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,466 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:14:20,467 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec15ac6edd88,45395,1733321660402 from backup master directory 2024-12-04T14:14:20,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:14:20,468 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:14:20,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:14:20,468 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:14:20,474 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec15ac6edd88:45395 2024-12-04T14:14:20,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:14:20,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:14:20,479 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/hbase.id with ID: 4df8eb57-9f7e-482f-aea2-0b0c62244cce 2024-12-04T14:14:20,492 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:14:20,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:14:20,505 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:14:20,506 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:14:20,506 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:14:20,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:14:20,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:14:20,518 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store 2024-12-04T14:14:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:14:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:14:20,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:20,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:14:20,525 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:20,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:20,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:14:20,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:20,525 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:14:20,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:14:20,526 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/.initializing 2024-12-04T14:14:20,526 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,530 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C45395%2C1733321660402, suffix=, logDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402, archiveDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/oldWALs, maxLogs=10 2024-12-04T14:14:20,530 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C45395%2C1733321660402.1733321660530 2024-12-04T14:14:20,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:20,538 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 2024-12-04T14:14:20,538 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:35407:35407)] 2024-12-04T14:14:20,538 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:14:20,538 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:20,538 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,538 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:14:20,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:20,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:14:20,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:14:20,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:14:20,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:14:20,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:14:20,547 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:14:20,548 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,549 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,551 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:14:20,553 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:14:20,555 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:14:20,556 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709883, jitterRate=-0.0973382443189621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:14:20,556 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:14:20,557 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:14:20,561 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475508e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:14:20,562 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-04T14:14:20,562 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:14:20,562 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:14:20,562 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:14:20,563 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T14:14:20,563 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-04T14:14:20,563 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:14:20,565 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:14:20,566 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:14:20,568 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:14:20,569 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:14:20,569 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:14:20,571 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:14:20,571 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:14:20,572 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:14:20,573 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:14:20,574 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:14:20,576 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:14:20,577 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:14:20,578 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:14:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:14:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:14:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,581 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ec15ac6edd88,45395,1733321660402, sessionid=0x1005d6150c90000, setting cluster-up flag (Was=false) 2024-12-04T14:14:20,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,590 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:14:20,591 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:20,600 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:14:20,601 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:20,603 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-04T14:14:20,604 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-04T14:14:20,604 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec15ac6edd88,45395,1733321660402 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec15ac6edd88:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:14:20,604 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,606 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:14:20,606 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733321690607 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:14:20,607 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:14:20,607 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,607 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:14:20,608 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,612 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:14:20,612 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:14:20,612 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:14:20,613 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:14:20,613 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:14:20,613 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321660613,5,FailOnTimeoutGroup] 2024-12-04T14:14:20,614 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321660613,5,FailOnTimeoutGroup] 2024-12-04T14:14:20,614 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,614 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:14:20,614 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,614 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:14:20,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:14:20,621 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-04T14:14:20,622 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4 2024-12-04T14:14:20,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:14:20,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:14:20,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:20,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:14:20,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:14:20,635 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:20,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:14:20,637 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:14:20,638 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:20,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:14:20,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:14:20,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:20,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:20,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740 2024-12-04T14:14:20,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740 2024-12-04T14:14:20,643 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:14:20,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:14:20,647 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:14:20,647 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=744394, jitterRate=-0.0534546822309494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:14:20,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:14:20,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:14:20,647 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:14:20,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:14:20,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:14:20,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:14:20,648 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:14:20,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:14:20,649 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:14:20,649 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-04T14:14:20,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:14:20,651 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:14:20,652 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:14:20,674 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec15ac6edd88:42781 2024-12-04T14:14:20,676 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1008): ClusterId : 4df8eb57-9f7e-482f-aea2-0b0c62244cce 2024-12-04T14:14:20,676 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:14:20,679 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:14:20,679 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:14:20,681 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:14:20,681 DEBUG [RS:0;ec15ac6edd88:42781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64db33d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:14:20,682 DEBUG [RS:0;ec15ac6edd88:42781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612495c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:14:20,682 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:14:20,682 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:14:20,682 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:14:20,683 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,45395,1733321660402 with isa=ec15ac6edd88/172.17.0.2:42781, startcode=1733321660448 2024-12-04T14:14:20,683 DEBUG [RS:0;ec15ac6edd88:42781 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:14:20,685 INFO [RS-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49699, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:14:20,686 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45395 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:20,686 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45395 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:20,688 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4 2024-12-04T14:14:20,688 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37875 2024-12-04T14:14:20,688 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:14:20,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:14:20,690 DEBUG [RS:0;ec15ac6edd88:42781 {}] zookeeper.ZKUtil(111): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:20,690 WARN [RS:0;ec15ac6edd88:42781 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:14:20,691 INFO [RS:0;ec15ac6edd88:42781 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:14:20,691 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:20,691 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,42781,1733321660448] 2024-12-04T14:14:20,694 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:14:20,694 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:14:20,696 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:14:20,697 INFO [RS:0;ec15ac6edd88:42781 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:14:20,697 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,697 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:14:20,698 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,698 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,698 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,698 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:14:20,699 DEBUG [RS:0;ec15ac6edd88:42781 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:14:20,702 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,702 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,702 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,702 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,702 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,42781,1733321660448-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:14:20,725 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:14:20,725 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,42781,1733321660448-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:20,748 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.Replication(204): ec15ac6edd88,42781,1733321660448 started 2024-12-04T14:14:20,748 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,42781,1733321660448, RpcServer on ec15ac6edd88/172.17.0.2:42781, sessionid=0x1005d6150c90001 2024-12-04T14:14:20,748 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:14:20,748 DEBUG [RS:0;ec15ac6edd88:42781 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:20,748 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,42781,1733321660448' 2024-12-04T14:14:20,748 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:14:20,749 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:14:20,749 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:14:20,749 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:14:20,749 DEBUG [RS:0;ec15ac6edd88:42781 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:20,749 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,42781,1733321660448' 2024-12-04T14:14:20,749 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:14:20,750 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:14:20,750 DEBUG [RS:0;ec15ac6edd88:42781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:14:20,750 INFO [RS:0;ec15ac6edd88:42781 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:14:20,750 INFO [RS:0;ec15ac6edd88:42781 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:14:20,802 WARN [ec15ac6edd88:45395 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-04T14:14:20,852 INFO [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C42781%2C1733321660448, suffix=, logDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448, archiveDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/oldWALs, maxLogs=32 2024-12-04T14:14:20,854 INFO [RS:0;ec15ac6edd88:42781 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:14:20,860 INFO [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:14:20,860 DEBUG [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35407:35407),(127.0.0.1/127.0.0.1:36305:36305)] 2024-12-04T14:14:21,052 DEBUG [ec15ac6edd88:45395 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:14:21,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,054 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,42781,1733321660448, state=OPENING 2024-12-04T14:14:21,056 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:14:21,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:21,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:21,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ec15ac6edd88,42781,1733321660448}] 2024-12-04T14:14:21,058 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:14:21,058 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:14:21,211 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,211 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:14:21,213 INFO [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46858, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:14:21,217 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-04T14:14:21,217 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:14:21,219 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C42781%2C1733321660448.meta, suffix=.meta, logDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448, archiveDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/oldWALs, maxLogs=32 2024-12-04T14:14:21,219 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta 2024-12-04T14:14:21,225 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta 2024-12-04T14:14:21,225 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:35407:35407)] 2024-12-04T14:14:21,225 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:14:21,226 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:14:21,226 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:14:21,226 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:14:21,226 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:14:21,226 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:21,226 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-04T14:14:21,226 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-04T14:14:21,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:14:21,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:14:21,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:21,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:21,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:14:21,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:14:21,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:21,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:21,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:14:21,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:14:21,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:21,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:14:21,232 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740 2024-12-04T14:14:21,234 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740 2024-12-04T14:14:21,235 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:14:21,237 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:14:21,238 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773003, jitterRate=-0.01707710325717926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:14:21,238 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:14:21,239 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733321661211 2024-12-04T14:14:21,241 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:14:21,241 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-04T14:14:21,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,242 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,42781,1733321660448, state=OPEN 2024-12-04T14:14:21,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:14:21,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:14:21,247 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:14:21,247 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:14:21,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:14:21,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ec15ac6edd88,42781,1733321660448 in 189 msec 2024-12-04T14:14:21,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:14:21,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 600 msec 2024-12-04T14:14:21,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 649 msec 2024-12-04T14:14:21,253 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733321661253, completionTime=-1 2024-12-04T14:14:21,253 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:14:21,253 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-04T14:14:21,254 DEBUG [hconnection-0x20d4a825-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:14:21,255 INFO [RS-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46870, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:14:21,256 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-04T14:14:21,256 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733321721256 2024-12-04T14:14:21,256 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733321781256 2024-12-04T14:14:21,256 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-04T14:14:21,262 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45395,1733321660402-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:21,262 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45395,1733321660402-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:21,262 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45395,1733321660402-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:21,262 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec15ac6edd88:45395, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:21,262 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:21,263 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-04T14:14:21,263 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:14:21,264 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-04T14:14:21,264 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-04T14:14:21,265 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:14:21,265 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:21,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:14:21,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:14:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:14:21,276 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b2c0585d44316e56d49ad84bc83a4b9e, NAME => 'hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4 2024-12-04T14:14:21,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:14:21,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:14:21,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:21,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing b2c0585d44316e56d49ad84bc83a4b9e, disabling compactions & flushes 2024-12-04T14:14:21,283 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. after waiting 0 ms 2024-12-04T14:14:21,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,283 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for b2c0585d44316e56d49ad84bc83a4b9e: 2024-12-04T14:14:21,284 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:14:21,284 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733321661284"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321661284"}]},"ts":"1733321661284"} 2024-12-04T14:14:21,286 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:14:21,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:14:21,288 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321661287"}]},"ts":"1733321661287"} 2024-12-04T14:14:21,289 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-04T14:14:21,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=b2c0585d44316e56d49ad84bc83a4b9e, ASSIGN}] 2024-12-04T14:14:21,294 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=b2c0585d44316e56d49ad84bc83a4b9e, ASSIGN 2024-12-04T14:14:21,295 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=b2c0585d44316e56d49ad84bc83a4b9e, ASSIGN; state=OFFLINE, location=ec15ac6edd88,42781,1733321660448; forceNewPlan=false, retain=false 2024-12-04T14:14:21,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=b2c0585d44316e56d49ad84bc83a4b9e, regionState=OPENING, regionLocation=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure b2c0585d44316e56d49ad84bc83a4b9e, server=ec15ac6edd88,42781,1733321660448}] 2024-12-04T14:14:21,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:21,600 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,605 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => b2c0585d44316e56d49ad84bc83a4b9e, NAME => 'hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:14:21,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:21,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,607 INFO [StoreOpener-b2c0585d44316e56d49ad84bc83a4b9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,609 INFO [StoreOpener-b2c0585d44316e56d49ad84bc83a4b9e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2c0585d44316e56d49ad84bc83a4b9e columnFamilyName info 2024-12-04T14:14:21,609 DEBUG [StoreOpener-b2c0585d44316e56d49ad84bc83a4b9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:21,609 INFO [StoreOpener-b2c0585d44316e56d49ad84bc83a4b9e-1 {}] regionserver.HStore(327): Store=b2c0585d44316e56d49ad84bc83a4b9e/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:14:21,610 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,610 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,612 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:14:21,614 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:14:21,615 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened b2c0585d44316e56d49ad84bc83a4b9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=820227, jitterRate=0.04297260940074921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:14:21,615 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for b2c0585d44316e56d49ad84bc83a4b9e: 2024-12-04T14:14:21,616 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e., pid=6, masterSystemTime=1733321661600 2024-12-04T14:14:21,617 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,618 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:14:21,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=b2c0585d44316e56d49ad84bc83a4b9e, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:14:21,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure b2c0585d44316e56d49ad84bc83a4b9e, server=ec15ac6edd88,42781,1733321660448 in 172 msec 2024-12-04T14:14:21,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:14:21,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=b2c0585d44316e56d49ad84bc83a4b9e, ASSIGN in 329 msec 2024-12-04T14:14:21,625 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:14:21,626 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321661626"}]},"ts":"1733321661626"} 2024-12-04T14:14:21,627 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-04T14:14:21,630 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:14:21,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 368 msec 2024-12-04T14:14:21,665 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-04T14:14:21,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:14:21,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:21,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:14:21,672 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-04T14:14:21,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:14:21,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 11 msec 2024-12-04T14:14:21,694 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-04T14:14:21,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:14:21,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec 2024-12-04T14:14:21,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-04T14:14:21,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.255sec 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45395,1733321660402-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:14:21,723 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45395,1733321660402-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:14:21,725 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:14:21,725 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:14:21,725 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45395,1733321660402-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:14:21,763 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4191fe07 to 127.0.0.1:59796 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60fc940c 2024-12-04T14:14:21,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e5b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:14:21,769 DEBUG [hconnection-0xd6f335-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:14:21,770 INFO [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46882, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:14:21,772 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ec15ac6edd88,45395,1733321660402 2024-12-04T14:14:21,772 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:14:21,776 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-04T14:14:21,776 INFO [Time-limited test {}] wal.TestLogRolling(297): Starting testLogRollOnPipelineRestart 2024-12-04T14:14:21,776 INFO [Time-limited test {}] wal.TestLogRolling(300): Replication=2 2024-12-04T14:14:21,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:14:21,778 INFO [RS-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48646, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:14:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T14:14:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T14:14:21,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:14:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T14:14:21,782 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:14:21,782 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:21,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 9 2024-12-04T14:14:21,783 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:14:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741837_1013 (size=395) 2024-12-04T14:14:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741837_1013 (size=395) 2024-12-04T14:14:21,792 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9edb346e6c7e8fb9bf2b040ed8795a7c, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4 2024-12-04T14:14:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35591 is added to blk_1073741838_1014 (size=78) 2024-12-04T14:14:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741838_1014 (size=78) 2024-12-04T14:14:21,802 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:21,802 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1681): Closing 9edb346e6c7e8fb9bf2b040ed8795a7c, disabling compactions & flushes 2024-12-04T14:14:21,802 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:21,802 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:21,802 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. after waiting 0 ms 2024-12-04T14:14:21,802 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:21,802 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:21,802 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9edb346e6c7e8fb9bf2b040ed8795a7c: 2024-12-04T14:14:21,804 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:14:21,804 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733321661804"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321661804"}]},"ts":"1733321661804"} 2024-12-04T14:14:21,806 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:14:21,807 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:14:21,807 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321661807"}]},"ts":"1733321661807"} 2024-12-04T14:14:21,809 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-12-04T14:14:21,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=9edb346e6c7e8fb9bf2b040ed8795a7c, ASSIGN}] 2024-12-04T14:14:21,815 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=9edb346e6c7e8fb9bf2b040ed8795a7c, ASSIGN 2024-12-04T14:14:21,816 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=9edb346e6c7e8fb9bf2b040ed8795a7c, ASSIGN; state=OFFLINE, location=ec15ac6edd88,42781,1733321660448; forceNewPlan=false, retain=false 2024-12-04T14:14:21,967 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=9edb346e6c7e8fb9bf2b040ed8795a7c, regionState=OPENING, regionLocation=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:21,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 9edb346e6c7e8fb9bf2b040ed8795a7c, server=ec15ac6edd88,42781,1733321660448}] 2024-12-04T14:14:22,121 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:22,125 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:22,125 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 9edb346e6c7e8fb9bf2b040ed8795a7c, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:14:22,126 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,126 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:14:22,126 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,126 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,127 INFO [StoreOpener-9edb346e6c7e8fb9bf2b040ed8795a7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,129 INFO [StoreOpener-9edb346e6c7e8fb9bf2b040ed8795a7c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9edb346e6c7e8fb9bf2b040ed8795a7c columnFamilyName info 2024-12-04T14:14:22,129 DEBUG [StoreOpener-9edb346e6c7e8fb9bf2b040ed8795a7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:14:22,130 INFO [StoreOpener-9edb346e6c7e8fb9bf2b040ed8795a7c-1 {}] regionserver.HStore(327): Store=9edb346e6c7e8fb9bf2b040ed8795a7c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:14:22,130 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,131 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,133 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:14:22,135 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:14:22,135 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 9edb346e6c7e8fb9bf2b040ed8795a7c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834281, jitterRate=0.06084367632865906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:14:22,136 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 9edb346e6c7e8fb9bf2b040ed8795a7c: 2024-12-04T14:14:22,137 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c., pid=11, masterSystemTime=1733321662121 2024-12-04T14:14:22,139 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:22,139 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:22,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=9edb346e6c7e8fb9bf2b040ed8795a7c, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,42781,1733321660448 2024-12-04T14:14:22,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T14:14:22,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 9edb346e6c7e8fb9bf2b040ed8795a7c, server=ec15ac6edd88,42781,1733321660448 in 173 msec 2024-12-04T14:14:22,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:14:22,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=9edb346e6c7e8fb9bf2b040ed8795a7c, ASSIGN in 330 msec 2024-12-04T14:14:22,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:14:22,147 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321662147"}]},"ts":"1733321662147"} 2024-12-04T14:14:22,149 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-12-04T14:14:22,151 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:14:22,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 372 msec 2024-12-04T14:14:22,532 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:23,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T14:14:23,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-12-04T14:14:23,240 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T14:14:23,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:24,534 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:25,534 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:26,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:26,729 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:14:26,742 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,742 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,743 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,743 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,743 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,744 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,747 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,749 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:14:26,764 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-12-04T14:14:27,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:28,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:29,537 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:30,537 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:31,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:14:31,785 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart, procId: 9 completed 2024-12-04T14:14:31,788 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T14:14:31,788 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:14:32,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:33,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:33,794 INFO [Time-limited test {}] wal.TestLogRolling(337): log.getCurrentFileName()): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:14:33,794 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:33,794 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:33,795 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:33,795 WARN [DataStreamer for file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 block BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK], DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]) is bad. 2024-12-04T14:14:33,795 WARN [DataStreamer for file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta block BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK], DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]) is bad. 2024-12-04T14:14:33,795 WARN [DataStreamer for file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 block BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK], DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37939,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]) is bad. 2024-12-04T14:14:33,796 WARN [PacketResponder: BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37939] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,796 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:54290 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37939:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54290 dst: /127.0.0.1:37939 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,796 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-541244016_22 at /127.0.0.1:50910 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50910 dst: /127.0.0.1:35591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,796 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:50936 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50936 dst: /127.0.0.1:35591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,795 WARN [PacketResponder: BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37939] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,797 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64139653{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:33,797 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-541244016_22 at /127.0.0.1:54252 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37939:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54252 dst: /127.0.0.1:37939 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,797 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3800cd12{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:33,797 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:54300 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37939:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54300 dst: /127.0.0.1:37939 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,797 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:33,798 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1794184d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:33,798 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4671d85f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:33,799 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:14:33,799 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339444668-172.17.0.2-1733321659710 (Datanode Uuid c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4) service to localhost/127.0.0.1:37875 2024-12-04T14:14:33,800 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data4/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:33,800 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:14:33,805 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data3/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:33,805 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:14:33,805 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:50950 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50950 dst: /127.0.0.1:35591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,805 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:14:33,815 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:33,818 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:33,819 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:33,819 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:33,819 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:14:33,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2cc8e486{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:33,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6be90b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:33,934 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9f7c69c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-34895-hadoop-hdfs-3_4_1-tests_jar-_-any-8553479192962671703/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:33,934 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fb87889{HTTP/1.1, (http/1.1)}{localhost:34895} 2024-12-04T14:14:33,934 INFO [Time-limited test {}] server.Server(415): Started @186405ms 2024-12-04T14:14:33,936 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:14:33,952 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:33,952 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:33,952 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1017 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1017 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:33,953 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:46172 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46172 dst: /127.0.0.1:35591 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,953 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:46190 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46190 dst: /127.0.0.1:35591 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,953 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-541244016_22 at /127.0.0.1:46188 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46188 dst: /127.0.0.1:35591 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:33,958 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5fd37f1f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:33,958 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e67e938{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:33,958 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:33,959 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72e01b7a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:33,959 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55633cc8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:33,960 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:14:33,960 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:14:33,960 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339444668-172.17.0.2-1733321659710 (Datanode Uuid 621646fd-ca1e-4f5a-b4ab-8821cf6e5161) service to localhost/127.0.0.1:37875 2024-12-04T14:14:33,960 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:14:33,962 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data1/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:33,963 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data2/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:33,963 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:14:33,972 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:33,974 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:33,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:33,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:33,975 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:14:33,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12566bf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:33,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@364be7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:34,036 WARN [Thread-1096 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:14:34,039 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a59b00705df48a1 with lease ID 0xb0988b48613d2a96: from storage DS-719143b2-faf0-4126-93d8-fd7ea76d9420 node DatanodeRegistration(127.0.0.1:37925, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=38833, infoSecurePort=0, ipcPort=34379, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:34,039 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a59b00705df48a1 with lease ID 0xb0988b48613d2a96: from storage DS-127d24c8-5c5a-446c-899d-09b0919f5f2a node DatanodeRegistration(127.0.0.1:37925, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=38833, infoSecurePort=0, ipcPort=34379, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:34,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6ab6befa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-45857-hadoop-hdfs-3_4_1-tests_jar-_-any-17115810883433101026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:34,098 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d115d2{HTTP/1.1, (http/1.1)}{localhost:45857} 2024-12-04T14:14:34,098 INFO [Time-limited test {}] server.Server(415): Started @186569ms 2024-12-04T14:14:34,100 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:14:34,189 WARN [Thread-1127 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:14:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1be4d5e062546272 with lease ID 0xb0988b48613d2a97: from storage DS-ff569953-6068-402f-bedf-7a0934dabfa9 node DatanodeRegistration(127.0.0.1:37081, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=39773, infoSecurePort=0, ipcPort=41159, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1be4d5e062546272 with lease ID 0xb0988b48613d2a97: from storage DS-77f15865-3968-43d7-a40e-aa5beceb926c node DatanodeRegistration(127.0.0.1:37081, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=39773, infoSecurePort=0, ipcPort=41159, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:34,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:35,117 INFO [Time-limited test {}] wal.TestLogRolling(349): Data Nodes restarted 2024-12-04T14:14:35,119 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-12-04T14:14:35,120 WARN [RS:0;ec15ac6edd88:42781.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=5, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:35,120 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C42781%2C1733321660448:(num 1733321660853) roll requested 2024-12-04T14:14:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42781 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:35,121 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:14:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42781 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46882 deadline: 1733321685120, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL 2024-12-04T14:14:35,126 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 newFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:14:35,127 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL 2024-12-04T14:14:35,127 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 with entries=5, filesize=2.09 KB; new WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:14:35,127 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38833:38833),(127.0.0.1/127.0.0.1:39773:39773)] 2024-12-04T14:14:35,127 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 is not closed yet, will try archiving it next time 2024-12-04T14:14:35,127 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:35,127 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:35,127 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:14:35,128 WARN [IPC Server handler 3 on default port 37875 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1015 2024-12-04T14:14:35,128 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 after 1ms 2024-12-04T14:14:35,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741833_1019 (size=2136) 2024-12-04T14:14:35,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:36,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:37,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:38,039 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T14:14:38,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:39,129 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 after 4002ms 2024-12-04T14:14:39,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:40,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:41,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:42,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:43,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:44,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:45,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:46,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:47,204 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-12-04T14:14:47,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:48,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:49,207 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018 java.io.IOException: Bad response ERROR for BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018 from datanode DatanodeInfoWithStorage[127.0.0.1:37081,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:49,207 WARN [DataStreamer for file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 block BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37925,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK], DatanodeInfoWithStorage[127.0.0.1:37081,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37081,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]) is bad. 2024-12-04T14:14:49,207 WARN [PacketResponder: BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37081] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:49,208 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:49484 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:37925:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49484 dst: /127.0.0.1:37925 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:49,208 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:33114 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:37081:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33114 dst: /127.0.0.1:37081 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:49,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6ab6befa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:49,210 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d115d2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:49,210 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:49,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@364be7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:49,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12566bf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:49,213 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:14:49,213 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:14:49,213 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339444668-172.17.0.2-1733321659710 (Datanode Uuid 621646fd-ca1e-4f5a-b4ab-8821cf6e5161) service to localhost/127.0.0.1:37875 2024-12-04T14:14:49,213 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:14:49,214 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data1/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:49,214 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data2/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:49,214 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:14:49,223 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:49,226 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:49,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:49,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:49,228 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:14:49,229 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46d21bf1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:49,229 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c191c0b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:49,345 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@56f9fb45{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-39501-hadoop-hdfs-3_4_1-tests_jar-_-any-596171429494286739/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:49,345 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6d1c0db4{HTTP/1.1, (http/1.1)}{localhost:39501} 2024-12-04T14:14:49,346 INFO [Time-limited test {}] server.Server(415): Started @201816ms 2024-12-04T14:14:49,347 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:14:49,364 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1020 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:49,365 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_860637845_22 at /127.0.0.1:47306 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:37925:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47306 dst: /127.0.0.1:37925 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:14:49,367 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9f7c69c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:49,368 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3fb87889{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:14:49,368 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:14:49,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6be90b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:14:49,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2cc8e486{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:14:49,370 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:14:49,370 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:14:49,370 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339444668-172.17.0.2-1733321659710 (Datanode Uuid c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4) service to localhost/127.0.0.1:37875 2024-12-04T14:14:49,370 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:14:49,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data3/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:49,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data4/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:14:49,371 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:14:49,382 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:14:49,386 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:14:49,388 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:14:49,388 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:14:49,388 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:14:49,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@767316b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:14:49,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3976e577{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:14:49,438 WARN [Thread-1171 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:14:49,441 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0d6381daba21768 with lease ID 0xb0988b48613d2a98: from storage DS-ff569953-6068-402f-bedf-7a0934dabfa9 node DatanodeRegistration(127.0.0.1:39175, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=43529, infoSecurePort=0, ipcPort=37083, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0d6381daba21768 with lease ID 0xb0988b48613d2a98: from storage DS-77f15865-3968-43d7-a40e-aa5beceb926c node DatanodeRegistration(127.0.0.1:39175, datanodeUuid=621646fd-ca1e-4f5a-b4ab-8821cf6e5161, infoPort=43529, infoSecurePort=0, ipcPort=37083, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:49,506 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bde5a8e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/java.io.tmpdir/jetty-localhost-41383-hadoop-hdfs-3_4_1-tests_jar-_-any-1329283350228240033/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:14:49,506 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f4fdc7d{HTTP/1.1, (http/1.1)}{localhost:41383} 2024-12-04T14:14:49,506 INFO [Time-limited test {}] server.Server(415): Started @201977ms 2024-12-04T14:14:49,507 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:14:49,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:49,597 WARN [Thread-1202 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:14:49,600 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2c36d119db87af1 with lease ID 0xb0988b48613d2a99: from storage DS-719143b2-faf0-4126-93d8-fd7ea76d9420 node DatanodeRegistration(127.0.0.1:43063, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=44431, infoSecurePort=0, ipcPort=43301, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:49,601 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2c36d119db87af1 with lease ID 0xb0988b48613d2a99: from storage DS-127d24c8-5c5a-446c-899d-09b0919f5f2a node DatanodeRegistration(127.0.0.1:43063, datanodeUuid=c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4, infoPort=44431, infoSecurePort=0, ipcPort=43301, storageInfo=lv=-57;cid=testClusterID;nsid=1329727300;c=1733321659710), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:14:50,384 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:14:50,528 INFO [Time-limited test {}] wal.TestLogRolling(366): Data Nodes restarted 2024-12-04T14:14:50,530 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-12-04T14:14:50,530 WARN [RS:0;ec15ac6edd88:42781.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=8, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37925,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,531 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C42781%2C1733321660448:(num 1733321675121) roll requested 2024-12-04T14:14:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42781 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37925,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,531 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.1733321690531 2024-12-04T14:14:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42781 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46882 deadline: 1733321700530, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL 2024-12-04T14:14:50,537 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 newFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 2024-12-04T14:14:50,537 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL 2024-12-04T14:14:50,537 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 2024-12-04T14:14:50,537 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43529:43529),(127.0.0.1/127.0.0.1:44431:44431)] 2024-12-04T14:14:50,537 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 is not closed yet, will try archiving it next time 2024-12-04T14:14:50,537 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37925,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,537 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37925,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,538 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:14:50,538 WARN [IPC Server handler 1 on default port 37875 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1020 2024-12-04T14:14:50,538 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 after 0ms 2024-12-04T14:14:50,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:50,607 WARN [master/ec15ac6edd88:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=95, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,608 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C45395%2C1733321660402:(num 1733321660530) roll requested 2024-12-04T14:14:50,608 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,608 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C45395%2C1733321660402.1733321690608 2024-12-04T14:14:50,608 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,613 WARN [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL 2024-12-04T14:14:50,614 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 with entries=92, filesize=45.98 KB; new WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321690608 2024-12-04T14:14:50,614 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43529:43529),(127.0.0.1/127.0.0.1:44431:44431)] 2024-12-04T14:14:50,614 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 is not closed yet, will try archiving it next time 2024-12-04T14:14:50,614 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,614 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:14:50,614 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 2024-12-04T14:14:50,614 WARN [IPC Server handler 4 on default port 37875 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741830_1017 2024-12-04T14:14:50,615 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 after 1ms 2024-12-04T14:14:51,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:52,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:53,442 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741839_1020: GenerationStamp not matched, existing replica is blk_1073741839_1018 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T14:14:53,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:53,682 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:14:53,683 INFO [RS-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:14:54,539 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 after 4001ms 2024-12-04T14:14:54,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:54,616 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402/ec15ac6edd88%2C45395%2C1733321660402.1733321660530 after 4002ms 2024-12-04T14:14:55,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:56,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:56,601 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1017: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T14:14:57,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:58,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:14:59,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:00,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:01,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:01,726 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:15:01,726 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:15:02,539 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:02,545 DEBUG [Time-limited test {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 newFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:02,547 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:02,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44431:44431),(127.0.0.1/127.0.0.1:43529:43529)] 2024-12-04T14:15:02,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 is not closed yet, will try archiving it next time 2024-12-04T14:15:02,547 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:15:02,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:15:02,548 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 after 1ms 2024-12-04T14:15:02,548 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:15:02,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741840_1021 (size=1264) 2024-12-04T14:15:02,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741840_1021 (size=1264) 2024-12-04T14:15:02,555 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733321661615/Put/vlen=162/seqid=0] 2024-12-04T14:15:02,556 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [default/info:d/1733321661676/Put/vlen=9/seqid=0] 2024-12-04T14:15:02,556 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #5: [hbase/info:d/1733321661699/Put/vlen=7/seqid=0] 2024-12-04T14:15:02,556 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733321662137/Put/vlen=218/seqid=0] 2024-12-04T14:15:02,556 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [row1002/info:/1733321671791/Put/vlen=1045/seqid=0] 2024-12-04T14:15:02,556 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321660853 2024-12-04T14:15:02,556 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:15:02,556 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:15:02,557 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 after 1ms 2024-12-04T14:15:02,557 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:15:02,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:02,560 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #6: [row1003/info:/1733321685201/Put/vlen=1045/seqid=0] 2024-12-04T14:15:02,560 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #7: [row1004/info:/1733321687205/Put/vlen=1045/seqid=0] 2024-12-04T14:15:02,560 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321675121 2024-12-04T14:15:02,560 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 2024-12-04T14:15:02,560 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 2024-12-04T14:15:02,560 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 after 0ms 2024-12-04T14:15:02,560 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321690531 2024-12-04T14:15:02,563 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #9: [row1005/info:/1733321700538/Put/vlen=1045/seqid=0] 2024-12-04T14:15:02,563 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:02,563 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:02,564 WARN [IPC Server handler 1 on default port 37875 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741842_1025 2024-12-04T14:15:02,564 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 after 1ms 2024-12-04T14:15:03,445 WARN [ResponseProcessor for block BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:03,445 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-541244016_22 at /127.0.0.1:38180 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:43063:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38180 dst: /127.0.0.1:43063 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43063 remote=/127.0.0.1:38180]. Total timeout mills is 60000, 59100 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:15:03,446 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-541244016_22 at /127.0.0.1:54688 [Receiving block BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:39175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54688 dst: /127.0.0.1:39175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:15:03,446 WARN [DataStreamer for file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 block BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43063,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK], DatanodeInfoWithStorage[127.0.0.1:39175,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43063,DS-719143b2-faf0-4126-93d8-fd7ea76d9420,DISK]) is bad. 2024-12-04T14:15:03,450 WARN [DataStreamer for file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 block BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:03,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741842_1026 (size=85) 2024-12-04T14:15:03,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:04,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:05,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:06,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:06,565 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 after 4002ms 2024-12-04T14:15:06,565 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:06,570 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:06,570 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.42 KB 2024-12-04T14:15:06,571 WARN [RS_OPEN_META-regionserver/ec15ac6edd88:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,571 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C42781%2C1733321660448.meta:.meta(num 1733321661219) roll requested 2024-12-04T14:15:06,571 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-04T14:15:06,571 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.meta.1733321706571.meta 2024-12-04T14:15:06,571 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,571 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing b2c0585d44316e56d49ad84bc83a4b9e 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-04T14:15:06,572 WARN [RS:0;ec15ac6edd88:42781.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=7, requesting roll of WAL org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,572 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for b2c0585d44316e56d49ad84bc83a4b9e: 2024-12-04T14:15:06,572 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=7, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=7, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,573 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 9edb346e6c7e8fb9bf2b040ed8795a7c 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-12-04T14:15:06,573 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 9edb346e6c7e8fb9bf2b040ed8795a7c: 2024-12-04T14:15:06,573 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=7, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=7, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,576 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-04T14:15:06,576 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL 2024-12-04T14:15:06,576 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-04T14:15:06,577 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta with entries=11, filesize=3.66 KB; new WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321706571.meta 2024-12-04T14:15:06,577 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4191fe07 to 127.0.0.1:59796 2024-12-04T14:15:06,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:15:06,577 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:15:06,577 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44431:44431),(127.0.0.1/127.0.0.1:43529:43529)] 2024-12-04T14:15:06,577 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1728269630, stopped=false 2024-12-04T14:15:06,577 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta is not closed yet, will try archiving it next time 2024-12-04T14:15:06,577 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ec15ac6edd88,45395,1733321660402 2024-12-04T14:15:06,577 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,577 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog ec15ac6edd88%2C42781%2C1733321660448:(num 1733321702539) roll requested 2024-12-04T14:15:06,577 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35591,DS-ff569953-6068-402f-bedf-7a0934dabfa9,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,577 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta 2024-12-04T14:15:06,577 INFO [regionserver/ec15ac6edd88:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42781%2C1733321660448.1733321706577 2024-12-04T14:15:06,578 WARN [IPC Server handler 2 on default port 37875 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta has not been closed. Lease recovery is in progress. RecoveryId = 1028 for block blk_1073741834_1016 2024-12-04T14:15:06,578 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta after 1ms 2024-12-04T14:15:06,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:15:06,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:15:06,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:06,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:06,580 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-04T14:15:06,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:15:06,581 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,42781,1733321660448' ***** 2024-12-04T14:15:06,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:15:06,581 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:15:06,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:15:06,582 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:15:06,582 INFO [RS:0;ec15ac6edd88:42781 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:15:06,582 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:15:06,582 INFO [RS:0;ec15ac6edd88:42781 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:15:06,582 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(3579): Received CLOSE for b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:15:06,583 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 newFile=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321706577 2024-12-04T14:15:06,583 WARN [regionserver/ec15ac6edd88:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=7, requesting roll of WAL 2024-12-04T14:15:06,583 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321706577 2024-12-04T14:15:06,584 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43529:43529),(127.0.0.1/127.0.0.1:44431:44431)] 2024-12-04T14:15:06,584 DEBUG [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 is not closed yet, will try archiving it next time 2024-12-04T14:15:06,584 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,584 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-339444668-172.17.0.2-1733321659710:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T14:15:06,585 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:06,585 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 after 0ms 2024-12-04T14:15:06,588 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(3579): Received CLOSE for 9edb346e6c7e8fb9bf2b040ed8795a7c 2024-12-04T14:15:06,588 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,42781,1733321660448 2024-12-04T14:15:06,588 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b2c0585d44316e56d49ad84bc83a4b9e, disabling compactions & flushes 2024-12-04T14:15:06,588 DEBUG [RS:0;ec15ac6edd88:42781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:15:06,588 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:15:06,588 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 to hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/oldWALs/ec15ac6edd88%2C42781%2C1733321660448.1733321702539 2024-12-04T14:15:06,588 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:15:06,588 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:15:06,588 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. after waiting 0 ms 2024-12-04T14:15:06,588 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:15:06,588 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:15:06,588 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:15:06,588 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b2c0585d44316e56d49ad84bc83a4b9e 1/1 column families, dataSize=78 B heapSize=728 B 2024-12-04T14:15:06,588 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:15:06,588 WARN [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-04T14:15:06,589 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-04T14:15:06,589 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, b2c0585d44316e56d49ad84bc83a4b9e=hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e., 9edb346e6c7e8fb9bf2b040ed8795a7c=TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c.} 2024-12-04T14:15:06,589 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:15:06,589 DEBUG [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 9edb346e6c7e8fb9bf2b040ed8795a7c, b2c0585d44316e56d49ad84bc83a4b9e 2024-12-04T14:15:06,589 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:15:06,589 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:15:06,589 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:15:06,589 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:15:06,589 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.89 KB 2024-12-04T14:15:06,589 WARN [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-04T14:15:06,589 WARN [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-04T14:15:06,605 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/.tmp/info/297c6049bf27425a8484378ff5e568e3 is 45, key is default/info:d/1733321661676/Put/seqid=0 2024-12-04T14:15:06,605 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/.tmp/info/c7c2d1e4c5204746bcb0d022ef9f4a44 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c./info:regioninfo/1733321662140/Put/seqid=0 2024-12-04T14:15:06,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741845_1030 (size=8268) 2024-12-04T14:15:06,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741846_1031 (size=5037) 2024-12-04T14:15:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741846_1031 (size=5037) 2024-12-04T14:15:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741845_1030 (size=8268) 2024-12-04T14:15:06,614 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/.tmp/info/c7c2d1e4c5204746bcb0d022ef9f4a44 2024-12-04T14:15:06,614 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/.tmp/info/297c6049bf27425a8484378ff5e568e3 2024-12-04T14:15:06,621 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/.tmp/info/297c6049bf27425a8484378ff5e568e3 as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/info/297c6049bf27425a8484378ff5e568e3 2024-12-04T14:15:06,627 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/info/297c6049bf27425a8484378ff5e568e3, entries=2, sequenceid=8, filesize=4.9 K 2024-12-04T14:15:06,628 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for b2c0585d44316e56d49ad84bc83a4b9e in 40ms, sequenceid=8, compaction requested=false 2024-12-04T14:15:06,633 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/namespace/b2c0585d44316e56d49ad84bc83a4b9e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T14:15:06,633 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:15:06,633 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b2c0585d44316e56d49ad84bc83a4b9e: 2024-12-04T14:15:06,634 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733321661263.b2c0585d44316e56d49ad84bc83a4b9e. 2024-12-04T14:15:06,634 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9edb346e6c7e8fb9bf2b040ed8795a7c, disabling compactions & flushes 2024-12-04T14:15:06,634 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:15:06,634 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:15:06,634 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. after waiting 0 ms 2024-12-04T14:15:06,634 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:15:06,634 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9edb346e6c7e8fb9bf2b040ed8795a7c 1/1 column families, dataSize=4.20 KB heapSize=4.98 KB 2024-12-04T14:15:06,634 WARN [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-04T14:15:06,638 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/.tmp/table/dfe7bfce8bf040388c96aa3f8c18f0a6 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733321662147/Put/seqid=0 2024-12-04T14:15:06,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741847_1032 (size=5482) 2024-12-04T14:15:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741847_1032 (size=5482) 2024-12-04T14:15:06,644 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=244 B at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/.tmp/table/dfe7bfce8bf040388c96aa3f8c18f0a6 2024-12-04T14:15:06,650 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/.tmp/info/c7c2d1e4c5204746bcb0d022ef9f4a44 as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/info/c7c2d1e4c5204746bcb0d022ef9f4a44 2024-12-04T14:15:06,656 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/info/c7c2d1e4c5204746bcb0d022ef9f4a44, entries=20, sequenceid=16, filesize=8.1 K 2024-12-04T14:15:06,657 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/.tmp/table/dfe7bfce8bf040388c96aa3f8c18f0a6 as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/table/dfe7bfce8bf040388c96aa3f8c18f0a6 2024-12-04T14:15:06,658 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/.tmp/info/71dbc2a2dd084dbc95542ee2165a748d is 1080, key is row1002/info:/1733321671791/Put/seqid=0 2024-12-04T14:15:06,663 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/table/dfe7bfce8bf040388c96aa3f8c18f0a6, entries=4, sequenceid=16, filesize=5.4 K 2024-12-04T14:15:06,664 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.90 KB/2972, heapSize ~5.14 KB/5264, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=16, compaction requested=false 2024-12-04T14:15:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741848_1033 (size=9270) 2024-12-04T14:15:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741848_1033 (size=9270) 2024-12-04T14:15:06,666 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/.tmp/info/71dbc2a2dd084dbc95542ee2165a748d 2024-12-04T14:15:06,671 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/.tmp/info/71dbc2a2dd084dbc95542ee2165a748d as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/info/71dbc2a2dd084dbc95542ee2165a748d 2024-12-04T14:15:06,677 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/info/71dbc2a2dd084dbc95542ee2165a748d, entries=4, sequenceid=12, filesize=9.1 K 2024-12-04T14:15:06,678 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 9edb346e6c7e8fb9bf2b040ed8795a7c in 44ms, sequenceid=12, compaction requested=false 2024-12-04T14:15:06,683 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/9edb346e6c7e8fb9bf2b040ed8795a7c/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2024-12-04T14:15:06,683 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/data/hbase/meta/1588230740/recovered.edits/19.seqid, newMaxSeqId=19, maxSeqId=1 2024-12-04T14:15:06,683 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:15:06,683 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9edb346e6c7e8fb9bf2b040ed8795a7c: 2024-12-04T14:15:06,683 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:15:06,684 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733321661779.9edb346e6c7e8fb9bf2b040ed8795a7c. 2024-12-04T14:15:06,684 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:15:06,684 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:15:06,684 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T14:15:06,704 INFO [regionserver/ec15ac6edd88:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:15:06,710 INFO [regionserver/ec15ac6edd88:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T14:15:06,710 INFO [regionserver/ec15ac6edd88:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T14:15:06,789 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,42781,1733321660448; all regions closed. 2024-12-04T14:15:06,790 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448 2024-12-04T14:15:06,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741843_1027 (size=761) 2024-12-04T14:15:06,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741843_1027 (size=761) 2024-12-04T14:15:07,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:08,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:08,603 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1016: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T14:15:09,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:10,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:10,579 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta after 4002ms 2024-12-04T14:15:10,579 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta to hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/oldWALs/ec15ac6edd88%2C42781%2C1733321660448.meta.1733321661219.meta 2024-12-04T14:15:10,582 DEBUG [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/oldWALs 2024-12-04T14:15:10,583 INFO [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C42781%2C1733321660448.meta:.meta(num 1733321706571) 2024-12-04T14:15:10,583 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/WALs/ec15ac6edd88,42781,1733321660448 2024-12-04T14:15:10,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741844_1029 (size=1979) 2024-12-04T14:15:10,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741844_1029 (size=1979) 2024-12-04T14:15:10,589 DEBUG [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(1071): Moved 4 WAL file(s) to /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/oldWALs 2024-12-04T14:15:10,589 INFO [RS:0;ec15ac6edd88:42781 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C42781%2C1733321660448:(num 1733321706577) 2024-12-04T14:15:10,589 DEBUG [RS:0;ec15ac6edd88:42781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:15:10,589 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:15:10,590 INFO [RS:0;ec15ac6edd88:42781 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T14:15:10,590 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:15:10,590 INFO [RS:0;ec15ac6edd88:42781 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42781 2024-12-04T14:15:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:15:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,42781,1733321660448 2024-12-04T14:15:10,593 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,42781,1733321660448] 2024-12-04T14:15:10,594 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,42781,1733321660448; numProcessing=1 2024-12-04T14:15:10,595 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,42781,1733321660448 already deleted, retry=false 2024-12-04T14:15:10,595 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,42781,1733321660448 expired; onlineServers=0 2024-12-04T14:15:10,595 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,45395,1733321660402' ***** 2024-12-04T14:15:10,595 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:15:10,595 DEBUG [M:0;ec15ac6edd88:45395 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689e47cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:15:10,595 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,45395,1733321660402 2024-12-04T14:15:10,595 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,45395,1733321660402; all regions closed. 2024-12-04T14:15:10,595 DEBUG [M:0;ec15ac6edd88:45395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:15:10,595 DEBUG [M:0;ec15ac6edd88:45395 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:15:10,595 DEBUG [M:0;ec15ac6edd88:45395 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:15:10,595 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:15:10,595 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321660613 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321660613,5,FailOnTimeoutGroup] 2024-12-04T14:15:10,595 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321660613 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321660613,5,FailOnTimeoutGroup] 2024-12-04T14:15:10,596 INFO [M:0;ec15ac6edd88:45395 {}] hbase.ChoreService(370): Chore service for: master/ec15ac6edd88:0 had [] on shutdown 2024-12-04T14:15:10,596 DEBUG [M:0;ec15ac6edd88:45395 {}] master.HMaster(1733): Stopping service threads 2024-12-04T14:15:10,596 INFO [M:0;ec15ac6edd88:45395 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:15:10,596 INFO [M:0;ec15ac6edd88:45395 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:15:10,596 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:15:10,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:15:10,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:10,597 DEBUG [M:0;ec15ac6edd88:45395 {}] zookeeper.ZKUtil(347): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:15:10,597 WARN [M:0;ec15ac6edd88:45395 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:15:10,597 INFO [M:0;ec15ac6edd88:45395 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-04T14:15:10,597 INFO [M:0;ec15ac6edd88:45395 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:15:10,597 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:15:10,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:15:10,597 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:10,597 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:10,597 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:15:10,597 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:10,597 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.23 KB 2024-12-04T14:15:10,613 DEBUG [M:0;ec15ac6edd88:45395 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/131e72deffe946299c95754655e281c3 is 82, key is hbase:meta,,1/info:regioninfo/1733321661241/Put/seqid=0 2024-12-04T14:15:10,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741849_1034 (size=5672) 2024-12-04T14:15:10,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741849_1034 (size=5672) 2024-12-04T14:15:10,619 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/131e72deffe946299c95754655e281c3 2024-12-04T14:15:10,639 DEBUG [M:0;ec15ac6edd88:45395 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/16dc90854271484e9f48cdc55309006c is 777, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733321662152/Put/seqid=0 2024-12-04T14:15:10,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741850_1035 (size=7468) 2024-12-04T14:15:10,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741850_1035 (size=7468) 2024-12-04T14:15:10,644 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.48 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/16dc90854271484e9f48cdc55309006c 2024-12-04T14:15:10,664 DEBUG [M:0;ec15ac6edd88:45395 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2414431917d143e6a430f0b3f3e78724 is 69, key is ec15ac6edd88,42781,1733321660448/rs:state/1733321660686/Put/seqid=0 2024-12-04T14:15:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741851_1036 (size=5156) 2024-12-04T14:15:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741851_1036 (size=5156) 2024-12-04T14:15:10,670 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2414431917d143e6a430f0b3f3e78724 2024-12-04T14:15:10,689 DEBUG [M:0;ec15ac6edd88:45395 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/27d1f895dd9547119e5059e406e4449c is 52, key is load_balancer_on/state:d/1733321661774/Put/seqid=0 2024-12-04T14:15:10,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:15:10,694 INFO [RS:0;ec15ac6edd88:42781 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,42781,1733321660448; zookeeper connection closed. 2024-12-04T14:15:10,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42781-0x1005d6150c90001, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:15:10,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741852_1037 (size=5056) 2024-12-04T14:15:10,695 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@191dfe19 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@191dfe19 2024-12-04T14:15:10,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741852_1037 (size=5056) 2024-12-04T14:15:10,695 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T14:15:10,695 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/27d1f895dd9547119e5059e406e4449c 2024-12-04T14:15:10,701 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/131e72deffe946299c95754655e281c3 as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/131e72deffe946299c95754655e281c3 2024-12-04T14:15:10,707 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/131e72deffe946299c95754655e281c3, entries=8, sequenceid=96, filesize=5.5 K 2024-12-04T14:15:10,708 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/16dc90854271484e9f48cdc55309006c as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/16dc90854271484e9f48cdc55309006c 2024-12-04T14:15:10,714 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/16dc90854271484e9f48cdc55309006c, entries=11, sequenceid=96, filesize=7.3 K 2024-12-04T14:15:10,715 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2414431917d143e6a430f0b3f3e78724 as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2414431917d143e6a430f0b3f3e78724 2024-12-04T14:15:10,721 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2414431917d143e6a430f0b3f3e78724, entries=1, sequenceid=96, filesize=5.0 K 2024-12-04T14:15:10,722 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/27d1f895dd9547119e5059e406e4449c as hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/27d1f895dd9547119e5059e406e4449c 2024-12-04T14:15:10,727 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37875/user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/27d1f895dd9547119e5059e406e4449c, entries=1, sequenceid=96, filesize=4.9 K 2024-12-04T14:15:10,728 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41040, heapSize ~49.16 KB/50344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=96, compaction requested=false 2024-12-04T14:15:10,730 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:10,730 DEBUG [M:0;ec15ac6edd88:45395 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:15:10,731 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/8e9c466c-99f2-e43c-bfe9-a51eeb3e27e4/MasterData/WALs/ec15ac6edd88,45395,1733321660402 2024-12-04T14:15:10,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43063 is added to blk_1073741841_1023 (size=757) 2024-12-04T14:15:10,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39175 is added to blk_1073741841_1023 (size=757) 2024-12-04T14:15:10,733 INFO [M:0;ec15ac6edd88:45395 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-04T14:15:10,733 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:15:10,733 INFO [M:0;ec15ac6edd88:45395 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45395 2024-12-04T14:15:10,736 DEBUG [M:0;ec15ac6edd88:45395 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ec15ac6edd88,45395,1733321660402 already deleted, retry=false 2024-12-04T14:15:10,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:15:10,839 INFO [M:0;ec15ac6edd88:45395 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,45395,1733321660402; zookeeper connection closed. 2024-12-04T14:15:10,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45395-0x1005d6150c90000, quorum=127.0.0.1:59796, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:15:10,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bde5a8e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:15:10,842 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f4fdc7d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:15:10,842 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:15:10,842 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3976e577{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:15:10,842 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@767316b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:15:10,844 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:15:10,844 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:15:10,845 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:15:10,845 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339444668-172.17.0.2-1733321659710 (Datanode Uuid c1b2d2c7-1e40-4fca-bf14-b2eedf5c20f4) service to localhost/127.0.0.1:37875 2024-12-04T14:15:10,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data3/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:15:10,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data4/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:15:10,846 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:15:10,848 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@56f9fb45{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:15:10,848 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d1c0db4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:15:10,848 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:15:10,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c191c0b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:15:10,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46d21bf1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:15:10,851 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:15:10,851 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:15:10,851 WARN [BP-339444668-172.17.0.2-1733321659710 heartbeating to localhost/127.0.0.1:37875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-339444668-172.17.0.2-1733321659710 (Datanode Uuid 621646fd-ca1e-4f5a-b4ab-8821cf6e5161) service to localhost/127.0.0.1:37875 2024-12-04T14:15:10,851 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:15:10,852 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data1/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:15:10,852 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/cluster_a98fe159-85b5-f6b1-b80b-d1d6ad4d81de/dfs/data/data2/current/BP-339444668-172.17.0.2-1733321659710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:15:10,852 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:15:10,859 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b9e6474{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:15:10,859 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@579b5c9d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:15:10,859 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:15:10,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c1c3841{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:15:10,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b73fa7a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir/,STOPPED} 2024-12-04T14:15:10,868 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:15:10,888 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-04T14:15:10,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=99 (was 85) Potentially hanging thread: nioEventLoopGroup-26-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37875 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37875 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-27-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-27-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37875 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37875 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37875 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:37875 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37875 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37875 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-27-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 428) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=144 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=6171 (was 6266) 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=99, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=144, ProcessCount=11, AvailableMemoryMB=6170 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.log.dir so I do NOT create it in target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3e50c409-ea02-e9e5-7a1f-c8c3e7d1fc4e/hadoop.tmp.dir so I do NOT create it in target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9, deleteOnExit=true 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/test.cache.data in system properties and HBase conf 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:15:10,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-04T14:15:10,903 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:15:10,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:15:10,916 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:15:10,984 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:15:10,989 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:15:10,990 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:15:10,990 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:15:10,990 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:15:10,991 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:15:10,991 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@233e6490{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:15:10,992 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4dc42901{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:15:11,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@57b58f32{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/java.io.tmpdir/jetty-localhost-39205-hadoop-hdfs-3_4_1-tests_jar-_-any-3719084802919379162/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:15:11,107 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15bb8b05{HTTP/1.1, (http/1.1)}{localhost:39205} 2024-12-04T14:15:11,108 INFO [Time-limited test {}] server.Server(415): Started @223578ms 2024-12-04T14:15:11,121 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:15:11,189 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:15:11,192 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:15:11,193 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:15:11,193 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:15:11,193 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:15:11,193 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d38e2e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:15:11,194 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c31c535{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:15:11,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12a372f6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/java.io.tmpdir/jetty-localhost-33213-hadoop-hdfs-3_4_1-tests_jar-_-any-12029257646936516225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:15:11,310 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@105e54da{HTTP/1.1, (http/1.1)}{localhost:33213} 2024-12-04T14:15:11,310 INFO [Time-limited test {}] server.Server(415): Started @223780ms 2024-12-04T14:15:11,311 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:15:11,350 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:15:11,353 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:15:11,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:15:11,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:15:11,355 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:15:11,355 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55215408{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:15:11,356 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a0d04be{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:15:11,410 WARN [Thread-1392 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data1/current/BP-995105679-172.17.0.2-1733321710934/current, will proceed with Du for space computation calculation, 2024-12-04T14:15:11,410 WARN [Thread-1393 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data2/current/BP-995105679-172.17.0.2-1733321710934/current, will proceed with Du for space computation calculation, 2024-12-04T14:15:11,432 WARN [Thread-1371 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:15:11,434 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3394ee4c92f0d3a with lease ID 0x3fc600d11d242794: Processing first storage report for DS-2f74c91e-30d8-4cf1-a33e-1df3e1d8d998 from datanode DatanodeRegistration(127.0.0.1:40727, datanodeUuid=e52ab209-c96c-4df7-ad4f-f36fedec8935, infoPort=41195, infoSecurePort=0, ipcPort=34381, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934) 2024-12-04T14:15:11,434 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3394ee4c92f0d3a with lease ID 0x3fc600d11d242794: from storage DS-2f74c91e-30d8-4cf1-a33e-1df3e1d8d998 node DatanodeRegistration(127.0.0.1:40727, datanodeUuid=e52ab209-c96c-4df7-ad4f-f36fedec8935, infoPort=41195, infoSecurePort=0, ipcPort=34381, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:15:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3394ee4c92f0d3a with lease ID 0x3fc600d11d242794: Processing first storage report for DS-0cf6d260-fa3e-4516-896d-c54ae7967fc1 from datanode DatanodeRegistration(127.0.0.1:40727, datanodeUuid=e52ab209-c96c-4df7-ad4f-f36fedec8935, infoPort=41195, infoSecurePort=0, ipcPort=34381, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934) 2024-12-04T14:15:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3394ee4c92f0d3a with lease ID 0x3fc600d11d242794: from storage DS-0cf6d260-fa3e-4516-896d-c54ae7967fc1 node DatanodeRegistration(127.0.0.1:40727, datanodeUuid=e52ab209-c96c-4df7-ad4f-f36fedec8935, infoPort=41195, infoSecurePort=0, ipcPort=34381, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:15:11,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54ce3781{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/java.io.tmpdir/jetty-localhost-37633-hadoop-hdfs-3_4_1-tests_jar-_-any-3077503981524484222/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:15:11,477 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4448dbfe{HTTP/1.1, (http/1.1)}{localhost:37633} 2024-12-04T14:15:11,477 INFO [Time-limited test {}] server.Server(415): Started @223948ms 2024-12-04T14:15:11,479 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:15:11,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:11,585 WARN [Thread-1419 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data4/current/BP-995105679-172.17.0.2-1733321710934/current, will proceed with Du for space computation calculation, 2024-12-04T14:15:11,585 WARN [Thread-1418 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data3/current/BP-995105679-172.17.0.2-1733321710934/current, will proceed with Du for space computation calculation, 2024-12-04T14:15:11,611 WARN [Thread-1407 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:15:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70d1b41de2df2c04 with lease ID 0x3fc600d11d242795: Processing first storage report for DS-edc45173-f8f8-4021-be5f-743d150cc145 from datanode DatanodeRegistration(127.0.0.1:37683, datanodeUuid=2d024fe1-5fed-4c6a-a2dc-faae988056c2, infoPort=34077, infoSecurePort=0, ipcPort=35639, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934) 2024-12-04T14:15:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70d1b41de2df2c04 with lease ID 0x3fc600d11d242795: from storage DS-edc45173-f8f8-4021-be5f-743d150cc145 node DatanodeRegistration(127.0.0.1:37683, datanodeUuid=2d024fe1-5fed-4c6a-a2dc-faae988056c2, infoPort=34077, infoSecurePort=0, ipcPort=35639, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:15:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70d1b41de2df2c04 with lease ID 0x3fc600d11d242795: Processing first storage report for DS-187da8cd-7ef6-41af-a361-f41a1c450f43 from datanode DatanodeRegistration(127.0.0.1:37683, datanodeUuid=2d024fe1-5fed-4c6a-a2dc-faae988056c2, infoPort=34077, infoSecurePort=0, ipcPort=35639, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934) 2024-12-04T14:15:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70d1b41de2df2c04 with lease ID 0x3fc600d11d242795: from storage DS-187da8cd-7ef6-41af-a361-f41a1c450f43 node DatanodeRegistration(127.0.0.1:37683, datanodeUuid=2d024fe1-5fed-4c6a-a2dc-faae988056c2, infoPort=34077, infoSecurePort=0, ipcPort=35639, storageInfo=lv=-57;cid=testClusterID;nsid=1899155727;c=1733321710934), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:15:11,617 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd 2024-12-04T14:15:11,619 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/zookeeper_0, clientPort=53928, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:15:11,620 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53928 2024-12-04T14:15:11,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,623 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:15:11,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:15:11,634 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524 with version=8 2024-12-04T14:15:11,634 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase-staging 2024-12-04T14:15:11,636 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:15:11,637 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:15:11,640 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46401 2024-12-04T14:15:11,640 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,644 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:46401 connecting to ZooKeeper ensemble=127.0.0.1:53928 2024-12-04T14:15:11,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464010x0, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:15:11,654 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,655 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,655 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46401-0x1005d6218e90000 connected 2024-12-04T14:15:11,655 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,655 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,661 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,661 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,661 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,663 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,669 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,669 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,670 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,670 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:11,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:15:11,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:15:11,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:15:11,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46401 2024-12-04T14:15:11,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46401 2024-12-04T14:15:11,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46401 2024-12-04T14:15:11,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46401 2024-12-04T14:15:11,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46401 2024-12-04T14:15:11,687 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524, hbase.cluster.distributed=false 2024-12-04T14:15:11,712 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:15:11,712 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:15:11,713 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42741 2024-12-04T14:15:11,714 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:15:11,716 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:15:11,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,718 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,721 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:42741 connecting to ZooKeeper ensemble=127.0.0.1:53928 2024-12-04T14:15:11,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427410x0, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:15:11,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427410x0, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:15:11,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427410x0, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:15:11,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427410x0, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:15:11,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42741-0x1005d6218e90001 connected 2024-12-04T14:15:11,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42741 2024-12-04T14:15:11,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42741 2024-12-04T14:15:11,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42741 2024-12-04T14:15:11,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42741 2024-12-04T14:15:11,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42741 2024-12-04T14:15:11,737 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:15:11,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:15:11,740 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:15:11,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:15:11,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,743 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:15:11,743 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec15ac6edd88,46401,1733321711636 from backup master directory 2024-12-04T14:15:11,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:15:11,745 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:15:11,745 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,755 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec15ac6edd88:46401 2024-12-04T14:15:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:15:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:15:11,763 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/hbase.id with ID: 8cbd68c6-3246-4a80-9975-892d1d616e75 2024-12-04T14:15:11,776 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:11,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:15:11,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:15:11,788 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:15:11,789 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:15:11,789 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:15:11,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:15:11,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:15:11,801 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store 2024-12-04T14:15:11,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:15:11,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:15:11,810 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:11,810 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:15:11,810 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:11,811 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:11,811 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:15:11,811 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:11,811 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:15:11,811 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:15:11,812 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/.initializing 2024-12-04T14:15:11,812 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/WALs/ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,815 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C46401%2C1733321711636, suffix=, logDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/WALs/ec15ac6edd88,46401,1733321711636, archiveDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/oldWALs, maxLogs=10 2024-12-04T14:15:11,816 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C46401%2C1733321711636.1733321711816 2024-12-04T14:15:11,824 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/WALs/ec15ac6edd88,46401,1733321711636/ec15ac6edd88%2C46401%2C1733321711636.1733321711816 2024-12-04T14:15:11,824 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34077:34077),(127.0.0.1/127.0.0.1:41195:41195)] 2024-12-04T14:15:11,824 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:15:11,824 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:11,824 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,824 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:15:11,830 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:11,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:15:11,832 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:15:11,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:15:11,834 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:15:11,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:15:11,836 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:15:11,837 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,838 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,840 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:15:11,841 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:15:11,843 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:15:11,844 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=822612, jitterRate=0.046005979180336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:15:11,844 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:15:11,845 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:15:11,848 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e25483, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:15:11,849 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-04T14:15:11,849 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:15:11,849 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:15:11,850 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:15:11,850 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T14:15:11,850 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-04T14:15:11,850 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:15:11,853 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:15:11,854 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:15:11,855 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:15:11,856 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:15:11,856 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:15:11,858 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:15:11,858 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:15:11,858 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:15:11,860 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:15:11,860 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:15:11,862 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:15:11,864 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:15:11,867 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:15:11,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:15:11,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:15:11,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,869 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ec15ac6edd88,46401,1733321711636, sessionid=0x1005d6218e90000, setting cluster-up flag (Was=false) 2024-12-04T14:15:11,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,877 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:15:11,878 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:11,886 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:15:11,886 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:11,889 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-04T14:15:11,889 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-04T14:15:11,889 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec15ac6edd88,46401,1733321711636 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec15ac6edd88:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:15:11,890 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,891 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733321741891 2024-12-04T14:15:11,891 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:15:11,892 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:15:11,892 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:15:11,892 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:15:11,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:15:11,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:15:11,893 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321711893,5,FailOnTimeoutGroup] 2024-12-04T14:15:11,893 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,893 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321711893,5,FailOnTimeoutGroup] 2024-12-04T14:15:11,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:15:11,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,893 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,893 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:15:11,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:15:11,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:15:11,900 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-04T14:15:11,900 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524 2024-12-04T14:15:11,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:15:11,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:15:11,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:11,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:15:11,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:15:11,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:11,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:15:11,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:15:11,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:11,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:15:11,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:15:11,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:11,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:11,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740 2024-12-04T14:15:11,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740 2024-12-04T14:15:11,917 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:15:11,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:15:11,920 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:15:11,921 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874937, jitterRate=0.11254005134105682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:15:11,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:15:11,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:15:11,921 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:15:11,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:15:11,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:15:11,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:15:11,922 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:15:11,922 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:15:11,923 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:15:11,923 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-04T14:15:11,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:15:11,924 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:15:11,925 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:15:11,949 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec15ac6edd88:42741 2024-12-04T14:15:11,950 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1008): ClusterId : 8cbd68c6-3246-4a80-9975-892d1d616e75 2024-12-04T14:15:11,951 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:15:11,953 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:15:11,953 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:15:11,955 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:15:11,955 DEBUG [RS:0;ec15ac6edd88:42741 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b00fab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:15:11,956 DEBUG [RS:0;ec15ac6edd88:42741 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e407096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:15:11,956 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:15:11,956 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:15:11,956 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:15:11,956 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,46401,1733321711636 with isa=ec15ac6edd88/172.17.0.2:42741, startcode=1733321711711 2024-12-04T14:15:11,956 DEBUG [RS:0;ec15ac6edd88:42741 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:15:11,958 INFO [RS-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47391, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:15:11,959 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46401 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:11,959 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46401 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:11,960 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524 2024-12-04T14:15:11,960 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:39351 2024-12-04T14:15:11,961 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:15:11,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:15:11,962 DEBUG [RS:0;ec15ac6edd88:42741 {}] zookeeper.ZKUtil(111): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:11,962 WARN [RS:0;ec15ac6edd88:42741 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:15:11,963 INFO [RS:0;ec15ac6edd88:42741 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:15:11,963 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:11,963 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,42741,1733321711711] 2024-12-04T14:15:11,966 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:15:11,966 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:15:11,968 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:15:11,968 INFO [RS:0;ec15ac6edd88:42741 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:15:11,968 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,968 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:15:11,969 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,969 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,969 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,969 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,969 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:15:11,970 DEBUG [RS:0;ec15ac6edd88:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:15:11,972 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,972 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,972 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,972 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:11,972 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,42741,1733321711711-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:15:11,986 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:15:11,986 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,42741,1733321711711-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:12,001 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.Replication(204): ec15ac6edd88,42741,1733321711711 started 2024-12-04T14:15:12,001 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,42741,1733321711711, RpcServer on ec15ac6edd88/172.17.0.2:42741, sessionid=0x1005d6218e90001 2024-12-04T14:15:12,001 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:15:12,001 DEBUG [RS:0;ec15ac6edd88:42741 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,001 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,42741,1733321711711' 2024-12-04T14:15:12,001 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:15:12,002 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:15:12,002 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:15:12,002 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:15:12,002 DEBUG [RS:0;ec15ac6edd88:42741 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,002 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,42741,1733321711711' 2024-12-04T14:15:12,002 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:15:12,003 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:15:12,003 DEBUG [RS:0;ec15ac6edd88:42741 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:15:12,003 INFO [RS:0;ec15ac6edd88:42741 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:15:12,003 INFO [RS:0;ec15ac6edd88:42741 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:15:12,075 WARN [ec15ac6edd88:46401 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-04T14:15:12,105 INFO [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C42741%2C1733321711711, suffix=, logDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711, archiveDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/oldWALs, maxLogs=32 2024-12-04T14:15:12,107 INFO [RS:0;ec15ac6edd88:42741 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42741%2C1733321711711.1733321712106 2024-12-04T14:15:12,117 INFO [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321712106 2024-12-04T14:15:12,117 DEBUG [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41195:41195),(127.0.0.1/127.0.0.1:34077:34077)] 2024-12-04T14:15:12,172 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:15:12,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,191 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,191 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,192 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:12,326 DEBUG [ec15ac6edd88:46401 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:15:12,326 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,327 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,42741,1733321711711, state=OPENING 2024-12-04T14:15:12,329 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:15:12,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:12,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:12,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:15:12,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ec15ac6edd88,42741,1733321711711}] 2024-12-04T14:15:12,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:15:12,483 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,484 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:15:12,486 INFO [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60136, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:15:12,489 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-04T14:15:12,490 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:15:12,491 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C42741%2C1733321711711.meta, suffix=.meta, logDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711, archiveDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/oldWALs, maxLogs=32 2024-12-04T14:15:12,492 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42741%2C1733321711711.meta.1733321712492.meta 2024-12-04T14:15:12,497 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.meta.1733321712492.meta 2024-12-04T14:15:12,497 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34077:34077),(127.0.0.1/127.0.0.1:41195:41195)] 2024-12-04T14:15:12,497 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:15:12,497 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:15:12,498 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:15:12,498 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:15:12,498 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:15:12,498 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:12,498 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-04T14:15:12,498 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-04T14:15:12,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:15:12,500 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:15:12,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:12,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:12,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:15:12,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:15:12,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:12,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:12,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:15:12,503 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:15:12,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:12,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:15:12,504 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740 2024-12-04T14:15:12,505 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740 2024-12-04T14:15:12,506 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:15:12,508 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:15:12,509 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=829217, jitterRate=0.05440501868724823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:15:12,509 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:15:12,510 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733321712483 2024-12-04T14:15:12,512 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:15:12,512 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-04T14:15:12,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,513 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,42741,1733321711711, state=OPEN 2024-12-04T14:15:12,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:15:12,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:15:12,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:15:12,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:15:12,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:15:12,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ec15ac6edd88,42741,1733321711711 in 187 msec 2024-12-04T14:15:12,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:15:12,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 597 msec 2024-12-04T14:15:12,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 635 msec 2024-12-04T14:15:12,524 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733321712524, completionTime=-1 2024-12-04T14:15:12,524 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:15:12,525 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-04T14:15:12,525 DEBUG [hconnection-0x50e2f914-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:15:12,527 INFO [RS-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60142, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:15:12,528 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-04T14:15:12,528 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733321772528 2024-12-04T14:15:12,528 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733321832528 2024-12-04T14:15:12,528 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec 2024-12-04T14:15:12,533 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321711636-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:12,534 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321711636-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:12,534 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321711636-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:12,534 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec15ac6edd88:46401, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:12,534 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:12,534 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-04T14:15:12,534 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:15:12,535 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-04T14:15:12,535 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-04T14:15:12,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:15:12,536 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:12,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:15:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:15:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:15:12,547 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 75446a4d6f5225fcb2b47994ef02dfe0, NAME => 'hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524 2024-12-04T14:15:12,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:15:12,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:15:12,559 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:12,559 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 75446a4d6f5225fcb2b47994ef02dfe0, disabling compactions & flushes 2024-12-04T14:15:12,559 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,559 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,559 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. after waiting 0 ms 2024-12-04T14:15:12,559 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,559 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,559 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 75446a4d6f5225fcb2b47994ef02dfe0: 2024-12-04T14:15:12,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:15:12,561 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733321712561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321712561"}]},"ts":"1733321712561"} 2024-12-04T14:15:12,563 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:15:12,564 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:15:12,565 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321712564"}]},"ts":"1733321712564"} 2024-12-04T14:15:12,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:12,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-04T14:15:12,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=75446a4d6f5225fcb2b47994ef02dfe0, ASSIGN}] 2024-12-04T14:15:12,572 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=75446a4d6f5225fcb2b47994ef02dfe0, ASSIGN 2024-12-04T14:15:12,573 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=75446a4d6f5225fcb2b47994ef02dfe0, ASSIGN; state=OFFLINE, location=ec15ac6edd88,42741,1733321711711; forceNewPlan=false, retain=false 2024-12-04T14:15:12,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=75446a4d6f5225fcb2b47994ef02dfe0, regionState=OPENING, regionLocation=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 75446a4d6f5225fcb2b47994ef02dfe0, server=ec15ac6edd88,42741,1733321711711}] 2024-12-04T14:15:12,879 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,883 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 75446a4d6f5225fcb2b47994ef02dfe0, NAME => 'hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:15:12,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:12,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,885 INFO [StoreOpener-75446a4d6f5225fcb2b47994ef02dfe0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,886 INFO [StoreOpener-75446a4d6f5225fcb2b47994ef02dfe0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75446a4d6f5225fcb2b47994ef02dfe0 columnFamilyName info 2024-12-04T14:15:12,886 DEBUG [StoreOpener-75446a4d6f5225fcb2b47994ef02dfe0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:12,887 INFO [StoreOpener-75446a4d6f5225fcb2b47994ef02dfe0-1 {}] regionserver.HStore(327): Store=75446a4d6f5225fcb2b47994ef02dfe0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:15:12,888 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,888 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:15:12,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:15:12,893 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 75446a4d6f5225fcb2b47994ef02dfe0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=689429, jitterRate=-0.123345747590065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:15:12,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 75446a4d6f5225fcb2b47994ef02dfe0: 2024-12-04T14:15:12,894 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0., pid=6, masterSystemTime=1733321712879 2024-12-04T14:15:12,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,896 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:12,897 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=75446a4d6f5225fcb2b47994ef02dfe0, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:12,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:15:12,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 75446a4d6f5225fcb2b47994ef02dfe0, server=ec15ac6edd88,42741,1733321711711 in 173 msec 2024-12-04T14:15:12,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:15:12,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=75446a4d6f5225fcb2b47994ef02dfe0, ASSIGN in 330 msec 2024-12-04T14:15:12,904 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:15:12,904 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321712904"}]},"ts":"1733321712904"} 2024-12-04T14:15:12,906 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-04T14:15:12,908 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:15:12,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 374 msec 2024-12-04T14:15:12,936 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-04T14:15:12,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:15:12,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:12,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:15:12,943 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-04T14:15:12,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:15:12,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 12 msec 2024-12-04T14:15:12,965 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-04T14:15:12,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:15:12,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 11 msec 2024-12-04T14:15:12,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-04T14:15:12,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.249sec 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321711636-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:15:12,994 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321711636-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:15:12,996 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:15:12,996 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:15:12,996 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,46401,1733321711636-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:15:13,040 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e0e3135 to 127.0.0.1:53928 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c97e8ff 2024-12-04T14:15:13,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c80a3c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:15:13,045 DEBUG [hconnection-0x2b0ca45f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:15:13,046 INFO [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53156, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:15:13,048 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ec15ac6edd88,46401,1733321711636 2024-12-04T14:15:13,048 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:15:13,050 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-04T14:15:13,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:15:13,053 INFO [RS-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:15:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T14:15:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T14:15:13,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:15:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:13,056 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:15:13,056 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:13,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 9 2024-12-04T14:15:13,057 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:15:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:15:13,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741837_1013 (size=405) 2024-12-04T14:15:13,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741837_1013 (size=405) 2024-12-04T14:15:13,068 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 456aaae8b81542c59f9a950ac0b44349, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524 2024-12-04T14:15:13,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741838_1014 (size=88) 2024-12-04T14:15:13,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741838_1014 (size=88) 2024-12-04T14:15:13,075 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:13,075 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1681): Closing 456aaae8b81542c59f9a950ac0b44349, disabling compactions & flushes 2024-12-04T14:15:13,075 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,075 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,075 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. after waiting 0 ms 2024-12-04T14:15:13,075 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,075 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,075 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:15:13,076 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:15:13,076 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733321713076"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321713076"}]},"ts":"1733321713076"} 2024-12-04T14:15:13,078 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:15:13,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:15:13,079 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321713079"}]},"ts":"1733321713079"} 2024-12-04T14:15:13,080 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-12-04T14:15:13,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=456aaae8b81542c59f9a950ac0b44349, ASSIGN}] 2024-12-04T14:15:13,085 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=456aaae8b81542c59f9a950ac0b44349, ASSIGN 2024-12-04T14:15:13,086 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=456aaae8b81542c59f9a950ac0b44349, ASSIGN; state=OFFLINE, location=ec15ac6edd88,42741,1733321711711; forceNewPlan=false, retain=false 2024-12-04T14:15:13,237 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=456aaae8b81542c59f9a950ac0b44349, regionState=OPENING, regionLocation=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:13,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 456aaae8b81542c59f9a950ac0b44349, server=ec15ac6edd88,42741,1733321711711}] 2024-12-04T14:15:13,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T14:15:13,391 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:13,396 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,396 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 456aaae8b81542c59f9a950ac0b44349, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:15:13,396 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,396 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:15:13,396 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,396 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,398 INFO [StoreOpener-456aaae8b81542c59f9a950ac0b44349-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,399 INFO [StoreOpener-456aaae8b81542c59f9a950ac0b44349-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 456aaae8b81542c59f9a950ac0b44349 columnFamilyName info 2024-12-04T14:15:13,399 DEBUG [StoreOpener-456aaae8b81542c59f9a950ac0b44349-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:15:13,400 INFO [StoreOpener-456aaae8b81542c59f9a950ac0b44349-1 {}] regionserver.HStore(327): Store=456aaae8b81542c59f9a950ac0b44349/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:15:13,400 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,401 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,403 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:15:13,405 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:15:13,406 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 456aaae8b81542c59f9a950ac0b44349; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=783431, jitterRate=-0.0038161873817443848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:15:13,407 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:15:13,408 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349., pid=11, masterSystemTime=1733321713391 2024-12-04T14:15:13,409 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,409 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:13,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=456aaae8b81542c59f9a950ac0b44349, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:13,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T14:15:13,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 456aaae8b81542c59f9a950ac0b44349, server=ec15ac6edd88,42741,1733321711711 in 174 msec 2024-12-04T14:15:13,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:15:13,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=456aaae8b81542c59f9a950ac0b44349, ASSIGN in 330 msec 2024-12-04T14:15:13,417 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:15:13,417 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321713417"}]},"ts":"1733321713417"} 2024-12-04T14:15:13,418 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-12-04T14:15:13,422 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:15:13,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 368 msec 2024-12-04T14:15:13,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:14,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:15,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:16,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:17,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:18,000 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:15:18,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,002 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,016 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,017 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,017 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,017 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,017 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,018 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,021 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,021 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,021 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:15:18,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T14:15:18,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-04T14:15:18,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-12-04T14:15:18,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:19,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:20,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:21,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:21,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta after 68042ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor238.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:15:22,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:15:23,059 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 9 completed 2024-12-04T14:15:23,062 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:23,062 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:23,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush hbase:namespace 2024-12-04T14:15:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace 2024-12-04T14:15:23,075 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-04T14:15:23,076 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:15:23,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:15:23,237 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:23,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42741 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-12-04T14:15:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:23,239 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 75446a4d6f5225fcb2b47994ef02dfe0 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-04T14:15:23,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:23,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T14:15:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/.tmp/info/8ec0c56367be42829ffc7260fc8f8133 is 45, key is default/info:d/1733321712947/Put/seqid=0 2024-12-04T14:15:23,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741839_1015 (size=5037) 2024-12-04T14:15:23,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741839_1015 (size=5037) 2024-12-04T14:15:23,261 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/.tmp/info/8ec0c56367be42829ffc7260fc8f8133 2024-12-04T14:15:23,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/.tmp/info/8ec0c56367be42829ffc7260fc8f8133 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/info/8ec0c56367be42829ffc7260fc8f8133 2024-12-04T14:15:23,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/info/8ec0c56367be42829ffc7260fc8f8133, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T14:15:23,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 75446a4d6f5225fcb2b47994ef02dfe0 in 35ms, sequenceid=6, compaction requested=false 2024-12-04T14:15:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 75446a4d6f5225fcb2b47994ef02dfe0: 2024-12-04T14:15:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:15:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13 2024-12-04T14:15:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster(4106): Remote procedure done, pid=13 2024-12-04T14:15:23,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-04T14:15:23,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 202 msec 2024-12-04T14:15:23,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace in 211 msec 2024-12-04T14:15:23,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:24,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:25,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:26,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:27,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:28,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:29,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:30,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:31,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:32,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-04T14:15:33,078 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: hbase:namespace, procId: 12 completed 2024-12-04T14:15:33,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-04T14:15:33,087 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:15:33,088 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:15:33,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:15:33,241 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:33,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42741 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-04T14:15:33,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:33,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing 456aaae8b81542c59f9a950ac0b44349 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T14:15:33,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/a7b8e708167e4eaea2a7a6436a460861 is 1080, key is row0001/info:/1733321733081/Put/seqid=0 2024-12-04T14:15:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741840_1016 (size=6033) 2024-12-04T14:15:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741840_1016 (size=6033) 2024-12-04T14:15:33,264 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/a7b8e708167e4eaea2a7a6436a460861 2024-12-04T14:15:33,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/a7b8e708167e4eaea2a7a6436a460861 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/a7b8e708167e4eaea2a7a6436a460861 2024-12-04T14:15:33,275 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/a7b8e708167e4eaea2a7a6436a460861, entries=1, sequenceid=5, filesize=5.9 K 2024-12-04T14:15:33,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 456aaae8b81542c59f9a950ac0b44349 in 34ms, sequenceid=5, compaction requested=false 2024-12-04T14:15:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:15:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-04T14:15:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster(4106): Remote procedure done, pid=15 2024-12-04T14:15:33,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T14:15:33,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 190 msec 2024-12-04T14:15:33,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec 2024-12-04T14:15:33,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:34,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:35,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:36,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:37,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:38,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:39,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:40,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:41,582 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:41,616 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:15:42,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-04T14:15:43,089 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 14 completed 2024-12-04T14:15:43,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:43,096 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:15:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-04T14:15:43,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:15:43,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:15:43,248 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:43,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42741 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-12-04T14:15:43,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:43,250 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing 456aaae8b81542c59f9a950ac0b44349 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T14:15:43,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/985778f2d655467cb9c063b85098b154 is 1080, key is row0002/info:/1733321743090/Put/seqid=0 2024-12-04T14:15:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741841_1017 (size=6033) 2024-12-04T14:15:43,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741841_1017 (size=6033) 2024-12-04T14:15:43,260 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/985778f2d655467cb9c063b85098b154 2024-12-04T14:15:43,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/985778f2d655467cb9c063b85098b154 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/985778f2d655467cb9c063b85098b154 2024-12-04T14:15:43,272 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/985778f2d655467cb9c063b85098b154, entries=1, sequenceid=9, filesize=5.9 K 2024-12-04T14:15:43,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 456aaae8b81542c59f9a950ac0b44349 in 23ms, sequenceid=9, compaction requested=false 2024-12-04T14:15:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:15:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17 2024-12-04T14:15:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster(4106): Remote procedure done, pid=17 2024-12-04T14:15:43,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T14:15:43,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec 2024-12-04T14:15:43,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec 2024-12-04T14:15:43,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:44,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:45,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:46,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:46,956 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:15:46,957 INFO [RS-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60342, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:15:47,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:48,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:49,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:50,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:51,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:52,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:53,000 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:15:53,000 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:15:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-04T14:15:53,097 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 16 completed 2024-12-04T14:15:53,100 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42741%2C1733321711711.1733321753099 2024-12-04T14:15:53,107 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321712106 with entries=13, filesize=6.41 KB; new WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321753099 2024-12-04T14:15:53,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34077:34077),(127.0.0.1/127.0.0.1:41195:41195)] 2024-12-04T14:15:53,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321712106 is not closed yet, will try archiving it next time 2024-12-04T14:15:53,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741833_1009 (size=6574) 2024-12-04T14:15:53,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741833_1009 (size=6574) 2024-12-04T14:15:53,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:15:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-04T14:15:53,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:15:53,114 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:15:53,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:15:53,266 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:15:53,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42741 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-12-04T14:15:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:53,267 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing 456aaae8b81542c59f9a950ac0b44349 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T14:15:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/5256eca114cd478b8b76616eabfd4a69 is 1080, key is row0003/info:/1733321753098/Put/seqid=0 2024-12-04T14:15:53,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741843_1019 (size=6033) 2024-12-04T14:15:53,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741843_1019 (size=6033) 2024-12-04T14:15:53,278 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/5256eca114cd478b8b76616eabfd4a69 2024-12-04T14:15:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/5256eca114cd478b8b76616eabfd4a69 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/5256eca114cd478b8b76616eabfd4a69 2024-12-04T14:15:53,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/5256eca114cd478b8b76616eabfd4a69, entries=1, sequenceid=13, filesize=5.9 K 2024-12-04T14:15:53,293 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 456aaae8b81542c59f9a950ac0b44349 in 26ms, sequenceid=13, compaction requested=true 2024-12-04T14:15:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:15:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:15:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-04T14:15:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster(4106): Remote procedure done, pid=19 2024-12-04T14:15:53,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-12-04T14:15:53,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec 2024-12-04T14:15:53,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec 2024-12-04T14:15:53,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:54,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:55,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:56,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:57,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:57,884 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 75446a4d6f5225fcb2b47994ef02dfe0, had cached 0 bytes from a total of 5037 2024-12-04T14:15:58,396 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 456aaae8b81542c59f9a950ac0b44349, had cached 0 bytes from a total of 18099 2024-12-04T14:15:58,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:15:59,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:00,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:01,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:02,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-04T14:16:03,115 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 18 completed 2024-12-04T14:16:03,115 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:16:03,116 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:16:03,117 DEBUG [Time-limited test {}] regionserver.HStore(1540): 456aaae8b81542c59f9a950ac0b44349/info is initiating minor compaction (all files) 2024-12-04T14:16:03,117 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:16:03,117 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:03,117 INFO [Time-limited test {}] regionserver.HRegion(2351): Starting compaction of 456aaae8b81542c59f9a950ac0b44349/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:03,117 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/a7b8e708167e4eaea2a7a6436a460861, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/985778f2d655467cb9c063b85098b154, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/5256eca114cd478b8b76616eabfd4a69] into tmpdir=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp, totalSize=17.7 K 2024-12-04T14:16:03,118 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting a7b8e708167e4eaea2a7a6436a460861, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733321733081 2024-12-04T14:16:03,118 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 985778f2d655467cb9c063b85098b154, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733321743090 2024-12-04T14:16:03,119 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 5256eca114cd478b8b76616eabfd4a69, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733321753098 2024-12-04T14:16:03,131 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 456aaae8b81542c59f9a950ac0b44349#info#compaction#32 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:16:03,132 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/f8951c74a07a4e20a1f9d4415a2f1103 is 1080, key is row0001/info:/1733321733081/Put/seqid=0 2024-12-04T14:16:03,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741844_1020 (size=8296) 2024-12-04T14:16:03,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741844_1020 (size=8296) 2024-12-04T14:16:03,143 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/f8951c74a07a4e20a1f9d4415a2f1103 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/f8951c74a07a4e20a1f9d4415a2f1103 2024-12-04T14:16:03,150 INFO [Time-limited test {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 456aaae8b81542c59f9a950ac0b44349/info of 456aaae8b81542c59f9a950ac0b44349 into f8951c74a07a4e20a1f9d4415a2f1103(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:16:03,150 DEBUG [Time-limited test {}] regionserver.HRegion(2381): Compaction status journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:16:03,152 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42741%2C1733321711711.1733321763152 2024-12-04T14:16:03,162 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321753099 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321763152 2024-12-04T14:16:03,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41195:41195),(127.0.0.1/127.0.0.1:34077:34077)] 2024-12-04T14:16:03,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321753099 is not closed yet, will try archiving it next time 2024-12-04T14:16:03,163 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321712106 to hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/oldWALs/ec15ac6edd88%2C42741%2C1733321711711.1733321712106 2024-12-04T14:16:03,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741842_1018 (size=2520) 2024-12-04T14:16:03,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741842_1018 (size=2520) 2024-12-04T14:16:03,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:16:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:16:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-04T14:16:03,168 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:16:03,169 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:16:03,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:16:03,321 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,42741,1733321711711 2024-12-04T14:16:03,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42741 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-04T14:16:03,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:03,322 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing 456aaae8b81542c59f9a950ac0b44349 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T14:16:03,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/2f2e39d9b92e4a40864ef25d565d4044 is 1080, key is row0000/info:/1733321763151/Put/seqid=0 2024-12-04T14:16:03,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741846_1022 (size=6033) 2024-12-04T14:16:03,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741846_1022 (size=6033) 2024-12-04T14:16:03,331 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/2f2e39d9b92e4a40864ef25d565d4044 2024-12-04T14:16:03,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/2f2e39d9b92e4a40864ef25d565d4044 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/2f2e39d9b92e4a40864ef25d565d4044 2024-12-04T14:16:03,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/2f2e39d9b92e4a40864ef25d565d4044, entries=1, sequenceid=18, filesize=5.9 K 2024-12-04T14:16:03,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 456aaae8b81542c59f9a950ac0b44349 in 22ms, sequenceid=18, compaction requested=false 2024-12-04T14:16:03,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:16:03,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:03,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21 2024-12-04T14:16:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.HMaster(4106): Remote procedure done, pid=21 2024-12-04T14:16:03,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20 2024-12-04T14:16:03,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 176 msec 2024-12-04T14:16:03,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 182 msec 2024-12-04T14:16:03,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:04,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:05,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:06,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:07,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:08,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:09,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:10,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:11,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:11,616 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:16:12,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:12,738 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 75446a4d6f5225fcb2b47994ef02dfe0 changed from -1.0 to 0.0, refreshing cache 2024-12-04T14:16:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46401 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-04T14:16:13,169 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 20 completed 2024-12-04T14:16:13,171 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C42741%2C1733321711711.1733321773171 2024-12-04T14:16:13,182 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321763152 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321773171 2024-12-04T14:16:13,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34077:34077),(127.0.0.1/127.0.0.1:41195:41195)] 2024-12-04T14:16:13,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321763152 is not closed yet, will try archiving it next time 2024-12-04T14:16:13,182 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711/ec15ac6edd88%2C42741%2C1733321711711.1733321753099 to hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/oldWALs/ec15ac6edd88%2C42741%2C1733321711711.1733321753099 2024-12-04T14:16:13,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-04T14:16:13,182 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-04T14:16:13,182 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e0e3135 to 127.0.0.1:53928 2024-12-04T14:16:13,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:16:13,183 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:16:13,183 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2068703845, stopped=false 2024-12-04T14:16:13,183 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ec15ac6edd88,46401,1733321711636 2024-12-04T14:16:13,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741845_1021 (size=2026) 2024-12-04T14:16:13,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741845_1021 (size=2026) 2024-12-04T14:16:13,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:16:13,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:16:13,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:13,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:13,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-04T14:16:13,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:16:13,185 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,42741,1733321711711' ***** 2024-12-04T14:16:13,185 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:16:13,185 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:16:13,185 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:16:13,186 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:16:13,186 INFO [RS:0;ec15ac6edd88:42741 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:16:13,186 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:16:13,186 INFO [RS:0;ec15ac6edd88:42741 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:16:13,186 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(3579): Received CLOSE for 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:16:13,186 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(3579): Received CLOSE for 456aaae8b81542c59f9a950ac0b44349 2024-12-04T14:16:13,187 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,42741,1733321711711 2024-12-04T14:16:13,187 DEBUG [RS:0;ec15ac6edd88:42741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 75446a4d6f5225fcb2b47994ef02dfe0, disabling compactions & flushes 2024-12-04T14:16:13,187 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:16:13,187 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:16:13,187 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:16:13,187 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. after waiting 0 ms 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:16:13,187 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:16:13,187 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-04T14:16:13,187 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 75446a4d6f5225fcb2b47994ef02dfe0=hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0., 456aaae8b81542c59f9a950ac0b44349=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.} 2024-12-04T14:16:13,187 DEBUG [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 456aaae8b81542c59f9a950ac0b44349, 75446a4d6f5225fcb2b47994ef02dfe0 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:16:13,187 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:16:13,187 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:16:13,187 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=3.05 KB heapSize=5.55 KB 2024-12-04T14:16:13,191 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/namespace/75446a4d6f5225fcb2b47994ef02dfe0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T14:16:13,192 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:16:13,192 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 75446a4d6f5225fcb2b47994ef02dfe0: 2024-12-04T14:16:13,192 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733321712534.75446a4d6f5225fcb2b47994ef02dfe0. 2024-12-04T14:16:13,192 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 456aaae8b81542c59f9a950ac0b44349, disabling compactions & flushes 2024-12-04T14:16:13,192 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:13,192 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:13,192 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. after waiting 0 ms 2024-12-04T14:16:13,192 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:13,193 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 456aaae8b81542c59f9a950ac0b44349 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T14:16:13,197 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/7baef1e0a5ac41f0902336899c7a4ca3 is 1080, key is row0001/info:/1733321773170/Put/seqid=0 2024-12-04T14:16:13,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741848_1024 (size=6033) 2024-12-04T14:16:13,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741848_1024 (size=6033) 2024-12-04T14:16:13,204 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/7baef1e0a5ac41f0902336899c7a4ca3 2024-12-04T14:16:13,208 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/.tmp/info/e2b5419ce8c94cd0bb84351155531b12 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349./info:regioninfo/1733321713410/Put/seqid=0 2024-12-04T14:16:13,214 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/.tmp/info/7baef1e0a5ac41f0902336899c7a4ca3 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/7baef1e0a5ac41f0902336899c7a4ca3 2024-12-04T14:16:13,219 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/7baef1e0a5ac41f0902336899c7a4ca3, entries=1, sequenceid=22, filesize=5.9 K 2024-12-04T14:16:13,221 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 456aaae8b81542c59f9a950ac0b44349 in 28ms, sequenceid=22, compaction requested=true 2024-12-04T14:16:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741849_1025 (size=8430) 2024-12-04T14:16:13,222 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/a7b8e708167e4eaea2a7a6436a460861, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/985778f2d655467cb9c063b85098b154, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/5256eca114cd478b8b76616eabfd4a69] to archive 2024-12-04T14:16:13,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741849_1025 (size=8430) 2024-12-04T14:16:13,222 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.79 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/.tmp/info/e2b5419ce8c94cd0bb84351155531b12 2024-12-04T14:16:13,223 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:16:13,225 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/a7b8e708167e4eaea2a7a6436a460861 to hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/a7b8e708167e4eaea2a7a6436a460861 2024-12-04T14:16:13,226 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/985778f2d655467cb9c063b85098b154 to hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/985778f2d655467cb9c063b85098b154 2024-12-04T14:16:13,228 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/5256eca114cd478b8b76616eabfd4a69 to hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/info/5256eca114cd478b8b76616eabfd4a69 2024-12-04T14:16:13,232 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/456aaae8b81542c59f9a950ac0b44349/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-12-04T14:16:13,233 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:13,233 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 456aaae8b81542c59f9a950ac0b44349: 2024-12-04T14:16:13,233 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733321713054.456aaae8b81542c59f9a950ac0b44349. 2024-12-04T14:16:13,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T14:16:13,242 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/.tmp/table/ddc896ce61874814807d709795377bef is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733321713417/Put/seqid=0 2024-12-04T14:16:13,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741850_1026 (size=5532) 2024-12-04T14:16:13,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741850_1026 (size=5532) 2024-12-04T14:16:13,248 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=264 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/.tmp/table/ddc896ce61874814807d709795377bef 2024-12-04T14:16:13,254 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/.tmp/info/e2b5419ce8c94cd0bb84351155531b12 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/info/e2b5419ce8c94cd0bb84351155531b12 2024-12-04T14:16:13,260 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/info/e2b5419ce8c94cd0bb84351155531b12, entries=20, sequenceid=14, filesize=8.2 K 2024-12-04T14:16:13,261 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/.tmp/table/ddc896ce61874814807d709795377bef as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/table/ddc896ce61874814807d709795377bef 2024-12-04T14:16:13,266 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/table/ddc896ce61874814807d709795377bef, entries=4, sequenceid=14, filesize=5.4 K 2024-12-04T14:16:13,267 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~3.05 KB/3122, heapSize ~5.27 KB/5400, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=14, compaction requested=false 2024-12-04T14:16:13,271 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1 2024-12-04T14:16:13,272 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:16:13,272 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:16:13,272 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:16:13,272 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T14:16:13,387 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,42741,1733321711711; all regions closed. 2024-12-04T14:16:13,388 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711 2024-12-04T14:16:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741834_1010 (size=4570) 2024-12-04T14:16:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741834_1010 (size=4570) 2024-12-04T14:16:13,392 DEBUG [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/oldWALs 2024-12-04T14:16:13,392 INFO [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C42741%2C1733321711711.meta:.meta(num 1733321712492) 2024-12-04T14:16:13,392 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/WALs/ec15ac6edd88,42741,1733321711711 2024-12-04T14:16:13,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741847_1023 (size=1545) 2024-12-04T14:16:13,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741847_1023 (size=1545) 2024-12-04T14:16:13,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:13,799 DEBUG [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(1071): Moved 2 WAL file(s) to /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/oldWALs 2024-12-04T14:16:13,799 INFO [RS:0;ec15ac6edd88:42741 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C42741%2C1733321711711:(num 1733321773171) 2024-12-04T14:16:13,799 DEBUG [RS:0;ec15ac6edd88:42741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:16:13,799 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:16:13,799 INFO [RS:0;ec15ac6edd88:42741 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T14:16:13,799 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:16:13,800 INFO [RS:0;ec15ac6edd88:42741 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42741 2024-12-04T14:16:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:16:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,42741,1733321711711 2024-12-04T14:16:13,804 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,42741,1733321711711] 2024-12-04T14:16:13,804 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,42741,1733321711711; numProcessing=1 2024-12-04T14:16:13,805 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,42741,1733321711711 already deleted, retry=false 2024-12-04T14:16:13,806 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,42741,1733321711711 expired; onlineServers=0 2024-12-04T14:16:13,806 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,46401,1733321711636' ***** 2024-12-04T14:16:13,806 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:16:13,806 DEBUG [M:0;ec15ac6edd88:46401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ff812c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:16:13,806 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,46401,1733321711636 2024-12-04T14:16:13,806 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,46401,1733321711636; all regions closed. 2024-12-04T14:16:13,806 DEBUG [M:0;ec15ac6edd88:46401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:16:13,806 DEBUG [M:0;ec15ac6edd88:46401 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:16:13,806 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:16:13,806 DEBUG [M:0;ec15ac6edd88:46401 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:16:13,806 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321711893 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321711893,5,FailOnTimeoutGroup] 2024-12-04T14:16:13,806 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321711893 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321711893,5,FailOnTimeoutGroup] 2024-12-04T14:16:13,806 INFO [M:0;ec15ac6edd88:46401 {}] hbase.ChoreService(370): Chore service for: master/ec15ac6edd88:0 had [] on shutdown 2024-12-04T14:16:13,806 DEBUG [M:0;ec15ac6edd88:46401 {}] master.HMaster(1733): Stopping service threads 2024-12-04T14:16:13,806 INFO [M:0;ec15ac6edd88:46401 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:16:13,807 INFO [M:0;ec15ac6edd88:46401 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:16:13,807 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:16:13,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:16:13,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:13,808 DEBUG [M:0;ec15ac6edd88:46401 {}] zookeeper.ZKUtil(347): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:16:13,808 WARN [M:0;ec15ac6edd88:46401 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:16:13,808 INFO [M:0;ec15ac6edd88:46401 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-04T14:16:13,808 INFO [M:0;ec15ac6edd88:46401 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:16:13,808 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:16:13,808 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:16:13,809 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:13,809 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:13,809 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:16:13,809 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:13,809 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=65.04 KB heapSize=81.64 KB 2024-12-04T14:16:13,827 DEBUG [M:0;ec15ac6edd88:46401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/490cf649fe1549a59a977e59da6df196 is 82, key is hbase:meta,,1/info:regioninfo/1733321712512/Put/seqid=0 2024-12-04T14:16:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741851_1027 (size=5672) 2024-12-04T14:16:13,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741851_1027 (size=5672) 2024-12-04T14:16:13,832 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/490cf649fe1549a59a977e59da6df196 2024-12-04T14:16:13,851 DEBUG [M:0;ec15ac6edd88:46401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b4006b1492014141a92c8318f77a6b3a is 798, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733321713423/Put/seqid=0 2024-12-04T14:16:13,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741852_1028 (size=8352) 2024-12-04T14:16:13,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741852_1028 (size=8352) 2024-12-04T14:16:13,856 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.44 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b4006b1492014141a92c8318f77a6b3a 2024-12-04T14:16:13,861 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b4006b1492014141a92c8318f77a6b3a 2024-12-04T14:16:13,882 DEBUG [M:0;ec15ac6edd88:46401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/df995a62cba84366adf589565bf4e996 is 69, key is ec15ac6edd88,42741,1733321711711/rs:state/1733321711959/Put/seqid=0 2024-12-04T14:16:13,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741853_1029 (size=5156) 2024-12-04T14:16:13,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741853_1029 (size=5156) 2024-12-04T14:16:13,888 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/df995a62cba84366adf589565bf4e996 2024-12-04T14:16:13,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:16:13,904 INFO [RS:0;ec15ac6edd88:42741 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,42741,1733321711711; zookeeper connection closed. 2024-12-04T14:16:13,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1005d6218e90001, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:16:13,904 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58484268 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58484268 2024-12-04T14:16:13,904 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T14:16:13,915 DEBUG [M:0;ec15ac6edd88:46401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/96decad213f84063801e059c8617bd64 is 52, key is load_balancer_on/state:d/1733321713049/Put/seqid=0 2024-12-04T14:16:13,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741854_1030 (size=5056) 2024-12-04T14:16:13,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741854_1030 (size=5056) 2024-12-04T14:16:13,920 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/96decad213f84063801e059c8617bd64 2024-12-04T14:16:13,926 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/490cf649fe1549a59a977e59da6df196 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/490cf649fe1549a59a977e59da6df196 2024-12-04T14:16:13,931 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/490cf649fe1549a59a977e59da6df196, entries=8, sequenceid=184, filesize=5.5 K 2024-12-04T14:16:13,932 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b4006b1492014141a92c8318f77a6b3a as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b4006b1492014141a92c8318f77a6b3a 2024-12-04T14:16:13,936 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b4006b1492014141a92c8318f77a6b3a 2024-12-04T14:16:13,936 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b4006b1492014141a92c8318f77a6b3a, entries=21, sequenceid=184, filesize=8.2 K 2024-12-04T14:16:13,937 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/df995a62cba84366adf589565bf4e996 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/df995a62cba84366adf589565bf4e996 2024-12-04T14:16:13,942 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/df995a62cba84366adf589565bf4e996, entries=1, sequenceid=184, filesize=5.0 K 2024-12-04T14:16:13,943 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/96decad213f84063801e059c8617bd64 as hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/96decad213f84063801e059c8617bd64 2024-12-04T14:16:13,948 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39351/user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/96decad213f84063801e059c8617bd64, entries=1, sequenceid=184, filesize=4.9 K 2024-12-04T14:16:13,949 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(3040): Finished flush of dataSize ~65.04 KB/66601, heapSize ~81.58 KB/83536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=184, compaction requested=false 2024-12-04T14:16:13,950 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:13,950 DEBUG [M:0;ec15ac6edd88:46401 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:16:13,950 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/92cee555-f9f8-9af9-d847-9294bd51b524/MasterData/WALs/ec15ac6edd88,46401,1733321711636 2024-12-04T14:16:13,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40727 is added to blk_1073741830_1006 (size=79122) 2024-12-04T14:16:13,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37683 is added to blk_1073741830_1006 (size=79122) 2024-12-04T14:16:13,953 INFO [M:0;ec15ac6edd88:46401 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-04T14:16:13,953 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:16:13,953 INFO [M:0;ec15ac6edd88:46401 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46401 2024-12-04T14:16:13,955 DEBUG [M:0;ec15ac6edd88:46401 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ec15ac6edd88,46401,1733321711636 already deleted, retry=false 2024-12-04T14:16:13,974 INFO [regionserver/ec15ac6edd88:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:16:14,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:16:14,057 INFO [M:0;ec15ac6edd88:46401 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,46401,1733321711636; zookeeper connection closed. 2024-12-04T14:16:14,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46401-0x1005d6218e90000, quorum=127.0.0.1:53928, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:16:14,060 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54ce3781{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:16:14,060 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4448dbfe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:16:14,060 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:16:14,060 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a0d04be{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:16:14,060 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55215408{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir/,STOPPED} 2024-12-04T14:16:14,062 WARN [BP-995105679-172.17.0.2-1733321710934 heartbeating to localhost/127.0.0.1:39351 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:16:14,062 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:16:14,062 WARN [BP-995105679-172.17.0.2-1733321710934 heartbeating to localhost/127.0.0.1:39351 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-995105679-172.17.0.2-1733321710934 (Datanode Uuid 2d024fe1-5fed-4c6a-a2dc-faae988056c2) service to localhost/127.0.0.1:39351 2024-12-04T14:16:14,062 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:16:14,063 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data3/current/BP-995105679-172.17.0.2-1733321710934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:16:14,063 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data4/current/BP-995105679-172.17.0.2-1733321710934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:16:14,063 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:16:14,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12a372f6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:16:14,065 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@105e54da{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:16:14,065 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:16:14,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c31c535{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:16:14,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d38e2e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir/,STOPPED} 2024-12-04T14:16:14,067 WARN [BP-995105679-172.17.0.2-1733321710934 heartbeating to localhost/127.0.0.1:39351 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:16:14,067 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:16:14,067 WARN [BP-995105679-172.17.0.2-1733321710934 heartbeating to localhost/127.0.0.1:39351 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-995105679-172.17.0.2-1733321710934 (Datanode Uuid e52ab209-c96c-4df7-ad4f-f36fedec8935) service to localhost/127.0.0.1:39351 2024-12-04T14:16:14,067 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:16:14,067 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data1/current/BP-995105679-172.17.0.2-1733321710934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:16:14,068 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/cluster_8aca4ca4-e377-e843-d5a4-662ed984bcf9/dfs/data/data2/current/BP-995105679-172.17.0.2-1733321710934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:16:14,068 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:16:14,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@57b58f32{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:16:14,074 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15bb8b05{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:16:14,074 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:16:14,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4dc42901{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:16:14,075 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@233e6490{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir/,STOPPED} 2024-12-04T14:16:14,081 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:16:14,099 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-04T14:16:14,107 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=107 (was 99) - Thread LEAK? -, OpenFileDescriptor=464 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=120 (was 144), ProcessCount=11 (was 11), AvailableMemoryMB=6022 (was 6170) 2024-12-04T14:16:14,114 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=108, OpenFileDescriptor=464, MaxFileDescriptor=1048576, SystemLoadAverage=120, ProcessCount=11, AvailableMemoryMB=6023 2024-12-04T14:16:14,114 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:16:14,114 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.log.dir so I do NOT create it in target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5 2024-12-04T14:16:14,114 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc8817fe-515a-7a24-daf0-8453dff9f9bd/hadoop.tmp.dir so I do NOT create it in target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5 2024-12-04T14:16:14,114 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2, deleteOnExit=true 2024-12-04T14:16:14,114 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/test.cache.data in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-04T14:16:14,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:16:14,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:16:14,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:16:14,129 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:16:14,193 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:16:14,197 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:16:14,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:16:14,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:16:14,198 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:16:14,199 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:16:14,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1297c717{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:16:14,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cad81e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:16:14,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34cb6972{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/java.io.tmpdir/jetty-localhost-41849-hadoop-hdfs-3_4_1-tests_jar-_-any-7637606123278751076/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:16:14,314 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@191bd0c3{HTTP/1.1, (http/1.1)}{localhost:41849} 2024-12-04T14:16:14,314 INFO [Time-limited test {}] server.Server(415): Started @286784ms 2024-12-04T14:16:14,327 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:16:14,379 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:16:14,382 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:16:14,383 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:16:14,383 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:16:14,383 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:16:14,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@555a9157{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:16:14,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@bf6e2c7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:16:14,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d11ec04{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/java.io.tmpdir/jetty-localhost-34551-hadoop-hdfs-3_4_1-tests_jar-_-any-978305264389057688/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:16:14,496 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17c78fce{HTTP/1.1, (http/1.1)}{localhost:34551} 2024-12-04T14:16:14,496 INFO [Time-limited test {}] server.Server(415): Started @286967ms 2024-12-04T14:16:14,498 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:16:14,527 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:16:14,531 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:16:14,532 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:16:14,532 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:16:14,532 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:16:14,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33dd8821{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:16:14,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4be9feb9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:16:14,600 WARN [Thread-1716 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data1/current/BP-832301905-172.17.0.2-1733321774141/current, will proceed with Du for space computation calculation, 2024-12-04T14:16:14,600 WARN [Thread-1717 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data2/current/BP-832301905-172.17.0.2-1733321774141/current, will proceed with Du for space computation calculation, 2024-12-04T14:16:14,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:14,623 WARN [Thread-1695 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:16:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9559a19d50a21a8 with lease ID 0x7b31eac1e740b147: Processing first storage report for DS-92dcf39f-e1d0-48d9-aa46-c097557bdeb8 from datanode DatanodeRegistration(127.0.0.1:41847, datanodeUuid=f67e5b87-0e40-4720-a29f-2036c21887c9, infoPort=37765, infoSecurePort=0, ipcPort=41903, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141) 2024-12-04T14:16:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9559a19d50a21a8 with lease ID 0x7b31eac1e740b147: from storage DS-92dcf39f-e1d0-48d9-aa46-c097557bdeb8 node DatanodeRegistration(127.0.0.1:41847, datanodeUuid=f67e5b87-0e40-4720-a29f-2036c21887c9, infoPort=37765, infoSecurePort=0, ipcPort=41903, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:16:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9559a19d50a21a8 with lease ID 0x7b31eac1e740b147: Processing first storage report for DS-44f17b54-ecb6-4b28-b65c-453e6f09c175 from datanode DatanodeRegistration(127.0.0.1:41847, datanodeUuid=f67e5b87-0e40-4720-a29f-2036c21887c9, infoPort=37765, infoSecurePort=0, ipcPort=41903, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141) 2024-12-04T14:16:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9559a19d50a21a8 with lease ID 0x7b31eac1e740b147: from storage DS-44f17b54-ecb6-4b28-b65c-453e6f09c175 node DatanodeRegistration(127.0.0.1:41847, datanodeUuid=f67e5b87-0e40-4720-a29f-2036c21887c9, infoPort=37765, infoSecurePort=0, ipcPort=41903, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:16:14,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@434b09e4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/java.io.tmpdir/jetty-localhost-39931-hadoop-hdfs-3_4_1-tests_jar-_-any-14977716341939487572/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:16:14,668 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17b5bb29{HTTP/1.1, (http/1.1)}{localhost:39931} 2024-12-04T14:16:14,668 INFO [Time-limited test {}] server.Server(415): Started @287138ms 2024-12-04T14:16:14,669 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:16:14,771 WARN [Thread-1742 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data3/current/BP-832301905-172.17.0.2-1733321774141/current, will proceed with Du for space computation calculation, 2024-12-04T14:16:14,771 WARN [Thread-1743 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data4/current/BP-832301905-172.17.0.2-1733321774141/current, will proceed with Du for space computation calculation, 2024-12-04T14:16:14,787 WARN [Thread-1731 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:16:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5a0158904c3928d4 with lease ID 0x7b31eac1e740b148: Processing first storage report for DS-82dbb713-5adb-48ba-b3bf-0bc6c376a84f from datanode DatanodeRegistration(127.0.0.1:36493, datanodeUuid=3c9ca7ad-63cf-4ea5-a044-6fda006a579f, infoPort=39193, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141) 2024-12-04T14:16:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5a0158904c3928d4 with lease ID 0x7b31eac1e740b148: from storage DS-82dbb713-5adb-48ba-b3bf-0bc6c376a84f node DatanodeRegistration(127.0.0.1:36493, datanodeUuid=3c9ca7ad-63cf-4ea5-a044-6fda006a579f, infoPort=39193, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:16:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5a0158904c3928d4 with lease ID 0x7b31eac1e740b148: Processing first storage report for DS-bc0991d1-3718-47a2-aea0-574bab1aeb20 from datanode DatanodeRegistration(127.0.0.1:36493, datanodeUuid=3c9ca7ad-63cf-4ea5-a044-6fda006a579f, infoPort=39193, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141) 2024-12-04T14:16:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5a0158904c3928d4 with lease ID 0x7b31eac1e740b148: from storage DS-bc0991d1-3718-47a2-aea0-574bab1aeb20 node DatanodeRegistration(127.0.0.1:36493, datanodeUuid=3c9ca7ad-63cf-4ea5-a044-6fda006a579f, infoPort=39193, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=894915515;c=1733321774141), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:16:14,791 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5 2024-12-04T14:16:14,794 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/zookeeper_0, clientPort=54338, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:16:14,795 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=54338 2024-12-04T14:16:14,795 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,796 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:16:14,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:16:14,811 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a with version=8 2024-12-04T14:16:14,811 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase-staging 2024-12-04T14:16:14,813 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:16:14,813 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:16:14,814 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40263 2024-12-04T14:16:14,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,815 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,817 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40263 connecting to ZooKeeper ensemble=127.0.0.1:54338 2024-12-04T14:16:14,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402630x0, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:16:14,824 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40263-0x1005d630fb00000 connected 2024-12-04T14:16:14,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:16:14,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:16:14,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:16:14,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40263 2024-12-04T14:16:14,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40263 2024-12-04T14:16:14,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40263 2024-12-04T14:16:14,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40263 2024-12-04T14:16:14,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40263 2024-12-04T14:16:14,848 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a, hbase.cluster.distributed=false 2024-12-04T14:16:14,864 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:16:14,864 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:16:14,865 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:37995 2024-12-04T14:16:14,865 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:16:14,868 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:16:14,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,871 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,875 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37995 connecting to ZooKeeper ensemble=127.0.0.1:54338 2024-12-04T14:16:14,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379950x0, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:16:14,879 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37995-0x1005d630fb00001 connected 2024-12-04T14:16:14,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:16:14,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:16:14,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:16:14,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-04T14:16:14,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37995 2024-12-04T14:16:14,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37995 2024-12-04T14:16:14,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-04T14:16:14,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-04T14:16:14,890 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:14,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:16:14,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:16:14,892 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:14,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:16:14,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:16:14,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,895 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:16:14,895 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec15ac6edd88,40263,1733321774812 from backup master directory 2024-12-04T14:16:14,896 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:16:14,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:14,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:16:14,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:16:14,897 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:16:14,897 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:14,908 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec15ac6edd88:40263 2024-12-04T14:16:14,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:16:14,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:16:14,912 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/hbase.id with ID: b296676a-6648-40c3-b74b-2e5a967f9fa0 2024-12-04T14:16:14,923 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:14,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:16:14,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:16:14,936 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:16:14,936 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:16:14,937 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:16:14,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:16:14,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:16:14,946 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store 2024-12-04T14:16:14,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:16:14,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:16:14,953 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:14,953 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:16:14,953 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:14,953 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:14,953 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:16:14,953 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:14,953 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:16:14,953 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:16:14,954 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/.initializing 2024-12-04T14:16:14,954 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/WALs/ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:14,956 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C40263%2C1733321774812, suffix=, logDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/WALs/ec15ac6edd88,40263,1733321774812, archiveDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/oldWALs, maxLogs=10 2024-12-04T14:16:14,957 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C40263%2C1733321774812.1733321774956 2024-12-04T14:16:14,961 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/WALs/ec15ac6edd88,40263,1733321774812/ec15ac6edd88%2C40263%2C1733321774812.1733321774956 2024-12-04T14:16:14,961 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39193:39193),(127.0.0.1/127.0.0.1:37765:37765)] 2024-12-04T14:16:14,961 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:16:14,961 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:14,961 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,961 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:16:14,964 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:14,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:14,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:16:14,965 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:14,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:14,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:16:14,967 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:14,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:14,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,968 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:16:14,968 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:14,968 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:14,969 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,969 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,971 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:16:14,972 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:16:14,974 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:16:14,974 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=696827, jitterRate=-0.11393871903419495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:16:14,975 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:16:14,975 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:16:14,978 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a3ec41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:16:14,979 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-04T14:16:14,979 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:16:14,979 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:16:14,979 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:16:14,980 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T14:16:14,980 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-04T14:16:14,980 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:16:14,981 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:16:14,982 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:16:14,983 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:16:14,984 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:16:14,984 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:16:14,985 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:16:14,985 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:16:14,986 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:16:14,988 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:16:14,988 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:16:14,989 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:16:14,990 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:16:14,992 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:16:14,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:16:14,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:16:14,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,993 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ec15ac6edd88,40263,1733321774812, sessionid=0x1005d630fb00000, setting cluster-up flag (Was=false) 2024-12-04T14:16:14,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:14,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:15,001 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:16:15,002 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:15,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:15,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:15,010 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:16:15,011 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:15,013 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-04T14:16:15,013 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-04T14:16:15,013 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:16:15,013 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec15ac6edd88,40263,1733321774812 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec15ac6edd88:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:16:15,014 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733321805014 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:16:15,015 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:16:15,015 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:16:15,015 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,016 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:16:15,016 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:16:15,016 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:16:15,016 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,016 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:16:15,016 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:16:15,016 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:16:15,019 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321775016,5,FailOnTimeoutGroup] 2024-12-04T14:16:15,024 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321775019,5,FailOnTimeoutGroup] 2024-12-04T14:16:15,024 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,024 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:16:15,024 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,024 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:16:15,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:16:15,030 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-04T14:16:15,030 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a 2024-12-04T14:16:15,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:16:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:16:15,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:15,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:16:15,039 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:16:15,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:15,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:16:15,040 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:16:15,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,041 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:15,041 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:16:15,042 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:16:15,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:15,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740 2024-12-04T14:16:15,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740 2024-12-04T14:16:15,045 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:16:15,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:16:15,048 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:16:15,048 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=758899, jitterRate=-0.035010337829589844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:16:15,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:16:15,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:16:15,048 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:16:15,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:16:15,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:16:15,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:16:15,049 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:16:15,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:16:15,050 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:16:15,050 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-04T14:16:15,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:16:15,051 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:16:15,051 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:16:15,102 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec15ac6edd88:37995 2024-12-04T14:16:15,103 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1008): ClusterId : b296676a-6648-40c3-b74b-2e5a967f9fa0 2024-12-04T14:16:15,103 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:16:15,105 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:16:15,105 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:16:15,106 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:16:15,107 DEBUG [RS:0;ec15ac6edd88:37995 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a53ccb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:16:15,107 DEBUG [RS:0;ec15ac6edd88:37995 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34678c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:16:15,107 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:16:15,107 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:16:15,107 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:16:15,108 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,40263,1733321774812 with isa=ec15ac6edd88/172.17.0.2:37995, startcode=1733321774863 2024-12-04T14:16:15,108 DEBUG [RS:0;ec15ac6edd88:37995 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:16:15,110 INFO [RS-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37767, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:16:15,111 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40263 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,111 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40263 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,112 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a 2024-12-04T14:16:15,112 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:39545 2024-12-04T14:16:15,112 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:16:15,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:16:15,114 DEBUG [RS:0;ec15ac6edd88:37995 {}] zookeeper.ZKUtil(111): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,114 WARN [RS:0;ec15ac6edd88:37995 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:16:15,114 INFO [RS:0;ec15ac6edd88:37995 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:16:15,114 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,37995,1733321774863] 2024-12-04T14:16:15,117 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:16:15,117 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:16:15,119 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:16:15,119 INFO [RS:0;ec15ac6edd88:37995 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:16:15,119 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,119 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:16:15,120 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:16:15,120 DEBUG [RS:0;ec15ac6edd88:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:16:15,121 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,121 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,121 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,121 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,121 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,37995,1733321774863-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:16:15,135 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:16:15,136 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,37995,1733321774863-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,150 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.Replication(204): ec15ac6edd88,37995,1733321774863 started 2024-12-04T14:16:15,150 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,37995,1733321774863, RpcServer on ec15ac6edd88/172.17.0.2:37995, sessionid=0x1005d630fb00001 2024-12-04T14:16:15,150 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:16:15,150 DEBUG [RS:0;ec15ac6edd88:37995 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,150 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,37995,1733321774863' 2024-12-04T14:16:15,150 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:16:15,150 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,37995,1733321774863' 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:16:15,151 DEBUG [RS:0;ec15ac6edd88:37995 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:16:15,151 INFO [RS:0;ec15ac6edd88:37995 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:16:15,151 INFO [RS:0;ec15ac6edd88:37995 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:16:15,202 WARN [ec15ac6edd88:40263 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-04T14:16:15,253 INFO [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C37995%2C1733321774863, suffix=, logDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863, archiveDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/oldWALs, maxLogs=32 2024-12-04T14:16:15,254 INFO [RS:0;ec15ac6edd88:37995 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C37995%2C1733321774863.1733321775254 2024-12-04T14:16:15,261 INFO [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321775254 2024-12-04T14:16:15,261 DEBUG [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39193:39193),(127.0.0.1/127.0.0.1:37765:37765)] 2024-12-04T14:16:15,452 DEBUG [ec15ac6edd88:40263 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:16:15,452 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,453 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,37995,1733321774863, state=OPENING 2024-12-04T14:16:15,455 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:16:15,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:15,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:15,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ec15ac6edd88,37995,1733321774863}] 2024-12-04T14:16:15,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:16:15,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:16:15,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:15,610 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,610 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:16:15,612 INFO [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42246, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:16:15,616 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-04T14:16:15,616 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:16:15,618 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C37995%2C1733321774863.meta, suffix=.meta, logDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863, archiveDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/oldWALs, maxLogs=32 2024-12-04T14:16:15,618 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C37995%2C1733321774863.meta.1733321775618.meta 2024-12-04T14:16:15,623 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.meta.1733321775618.meta 2024-12-04T14:16:15,623 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39193:39193),(127.0.0.1/127.0.0.1:37765:37765)] 2024-12-04T14:16:15,623 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:16:15,623 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:16:15,624 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:16:15,624 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:16:15,624 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:16:15,624 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:15,624 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-04T14:16:15,624 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-04T14:16:15,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:16:15,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:16:15,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:15,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:16:15,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:16:15,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:15,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:16:15,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:16:15,629 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:16:15,630 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740 2024-12-04T14:16:15,630 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740 2024-12-04T14:16:15,632 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:16:15,633 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:16:15,634 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800376, jitterRate=0.017731070518493652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:16:15,634 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:16:15,635 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733321775609 2024-12-04T14:16:15,637 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:16:15,637 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-04T14:16:15,637 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,638 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,37995,1733321774863, state=OPEN 2024-12-04T14:16:15,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:16:15,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:16:15,643 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:16:15,643 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:16:15,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:16:15,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ec15ac6edd88,37995,1733321774863 in 186 msec 2024-12-04T14:16:15,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:16:15,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 595 msec 2024-12-04T14:16:15,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 635 msec 2024-12-04T14:16:15,648 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733321775648, completionTime=-1 2024-12-04T14:16:15,648 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:16:15,648 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-04T14:16:15,652 DEBUG [hconnection-0x595b1db4-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:16:15,653 INFO [RS-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42262, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:16:15,654 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-04T14:16:15,654 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733321835654 2024-12-04T14:16:15,654 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733321895654 2024-12-04T14:16:15,654 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 5 msec 2024-12-04T14:16:15,659 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40263,1733321774812-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,659 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40263,1733321774812-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,659 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40263,1733321774812-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,660 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec15ac6edd88:40263, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,660 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:15,660 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-04T14:16:15,660 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:16:15,661 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-04T14:16:15,661 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-04T14:16:15,662 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:16:15,662 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:15,663 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:16:15,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:16:15,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:16:15,672 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8dc4e070b920b05210aa2083cb985207, NAME => 'hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a 2024-12-04T14:16:15,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:16:15,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:16:15,682 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:15,682 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 8dc4e070b920b05210aa2083cb985207, disabling compactions & flushes 2024-12-04T14:16:15,682 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:15,682 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:15,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. after waiting 0 ms 2024-12-04T14:16:15,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:15,683 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:15,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8dc4e070b920b05210aa2083cb985207: 2024-12-04T14:16:15,684 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:16:15,684 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733321775684"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321775684"}]},"ts":"1733321775684"} 2024-12-04T14:16:15,686 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:16:15,687 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:16:15,687 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321775687"}]},"ts":"1733321775687"} 2024-12-04T14:16:15,689 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-04T14:16:15,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8dc4e070b920b05210aa2083cb985207, ASSIGN}] 2024-12-04T14:16:15,694 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8dc4e070b920b05210aa2083cb985207, ASSIGN 2024-12-04T14:16:15,695 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8dc4e070b920b05210aa2083cb985207, ASSIGN; state=OFFLINE, location=ec15ac6edd88,37995,1733321774863; forceNewPlan=false, retain=false 2024-12-04T14:16:15,845 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8dc4e070b920b05210aa2083cb985207, regionState=OPENING, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:15,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8dc4e070b920b05210aa2083cb985207, server=ec15ac6edd88,37995,1733321774863}] 2024-12-04T14:16:16,000 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:16,004 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:16,004 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 8dc4e070b920b05210aa2083cb985207, NAME => 'hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:16:16,004 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,004 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:16,004 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,004 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,006 INFO [StoreOpener-8dc4e070b920b05210aa2083cb985207-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,007 INFO [StoreOpener-8dc4e070b920b05210aa2083cb985207-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dc4e070b920b05210aa2083cb985207 columnFamilyName info 2024-12-04T14:16:16,007 DEBUG [StoreOpener-8dc4e070b920b05210aa2083cb985207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:16,008 INFO [StoreOpener-8dc4e070b920b05210aa2083cb985207-1 {}] regionserver.HStore(327): Store=8dc4e070b920b05210aa2083cb985207/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:16,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,010 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 8dc4e070b920b05210aa2083cb985207 2024-12-04T14:16:16,012 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:16:16,013 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 8dc4e070b920b05210aa2083cb985207; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781104, jitterRate=-0.006775587797164917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:16:16,013 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 8dc4e070b920b05210aa2083cb985207: 2024-12-04T14:16:16,014 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207., pid=6, masterSystemTime=1733321776000 2024-12-04T14:16:16,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:16,016 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:16:16,016 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8dc4e070b920b05210aa2083cb985207, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:16,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:16:16,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8dc4e070b920b05210aa2083cb985207, server=ec15ac6edd88,37995,1733321774863 in 171 msec 2024-12-04T14:16:16,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:16:16,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8dc4e070b920b05210aa2083cb985207, ASSIGN in 327 msec 2024-12-04T14:16:16,022 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:16:16,023 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321776023"}]},"ts":"1733321776023"} 2024-12-04T14:16:16,024 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-04T14:16:16,027 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:16:16,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 367 msec 2024-12-04T14:16:16,062 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-04T14:16:16,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:16,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:16:16,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:16:16,069 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-04T14:16:16,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:16:16,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 10 msec 2024-12-04T14:16:16,090 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-04T14:16:16,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:16:16,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 9 msec 2024-12-04T14:16:16,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-04T14:16:16,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-04T14:16:16,109 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.212sec 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40263,1733321774812-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:16:16,110 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40263,1733321774812-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:16:16,112 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:16:16,112 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:16:16,112 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,40263,1733321774812-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:16:16,192 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7bfbd5bb to 127.0.0.1:54338 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5de2049e 2024-12-04T14:16:16,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620fc06d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:16:16,198 DEBUG [hconnection-0x12e1f3d6-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:16:16,200 INFO [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42268, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:16:16,201 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ec15ac6edd88,40263,1733321774812 2024-12-04T14:16:16,201 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:16:16,203 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-04T14:16:16,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:16:16,206 INFO [RS-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38254, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:16:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T14:16:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T14:16:16,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:16:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-12-04T14:16:16,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:16:16,209 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:16,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 9 2024-12-04T14:16:16,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:16:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:16:16,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741837_1013 (size=381) 2024-12-04T14:16:16,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741837_1013 (size=381) 2024-12-04T14:16:16,218 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 939b7947bcc0d24d524d3c093fe60a0d, NAME => 'TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a 2024-12-04T14:16:16,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741838_1014 (size=64) 2024-12-04T14:16:16,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741838_1014 (size=64) 2024-12-04T14:16:16,225 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:16,225 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 939b7947bcc0d24d524d3c093fe60a0d, disabling compactions & flushes 2024-12-04T14:16:16,226 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,226 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,226 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. after waiting 0 ms 2024-12-04T14:16:16,226 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,226 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,226 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:16,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:16:16,227 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733321776227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321776227"}]},"ts":"1733321776227"} 2024-12-04T14:16:16,229 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:16:16,230 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:16:16,230 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321776230"}]},"ts":"1733321776230"} 2024-12-04T14:16:16,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-12-04T14:16:16,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, ASSIGN}] 2024-12-04T14:16:16,237 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, ASSIGN 2024-12-04T14:16:16,238 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, ASSIGN; state=OFFLINE, location=ec15ac6edd88,37995,1733321774863; forceNewPlan=false, retain=false 2024-12-04T14:16:16,388 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=939b7947bcc0d24d524d3c093fe60a0d, regionState=OPENING, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:16,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863}] 2024-12-04T14:16:16,542 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:16,546 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,546 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 939b7947bcc0d24d524d3c093fe60a0d, NAME => 'TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:16:16,547 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,547 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:16,547 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,547 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,548 INFO [StoreOpener-939b7947bcc0d24d524d3c093fe60a0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,550 INFO [StoreOpener-939b7947bcc0d24d524d3c093fe60a0d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 939b7947bcc0d24d524d3c093fe60a0d columnFamilyName info 2024-12-04T14:16:16,550 DEBUG [StoreOpener-939b7947bcc0d24d524d3c093fe60a0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:16,550 INFO [StoreOpener-939b7947bcc0d24d524d3c093fe60a0d-1 {}] regionserver.HStore(327): Store=939b7947bcc0d24d524d3c093fe60a0d/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:16,551 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,552 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,553 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:16,556 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:16:16,556 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 939b7947bcc0d24d524d3c093fe60a0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=806056, jitterRate=0.024953216314315796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:16:16,557 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:16,558 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d., pid=11, masterSystemTime=1733321776542 2024-12-04T14:16:16,559 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,559 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:16,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=939b7947bcc0d24d524d3c093fe60a0d, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:16,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T14:16:16,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863 in 172 msec 2024-12-04T14:16:16,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:16:16,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, ASSIGN in 328 msec 2024-12-04T14:16:16,566 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:16:16,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321776566"}]},"ts":"1733321776566"} 2024-12-04T14:16:16,567 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-12-04T14:16:16,570 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:16:16,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRolling in 363 msec 2024-12-04T14:16:16,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:17,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:18,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,205 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,205 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,205 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,205 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,205 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,205 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,208 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,208 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,208 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,210 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:18,714 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:16:18,715 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,715 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,715 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,731 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,731 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,736 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:18,738 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:19,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:20,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:21,117 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T14:16:21,118 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-04T14:16:21,118 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-12-04T14:16:21,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:22,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:23,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-04T14:16:23,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T14:16:23,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:24,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:25,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40263 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-04T14:16:26,211 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling, procId: 9 completed 2024-12-04T14:16:26,214 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRolling 2024-12-04T14:16:26,214 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:26,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:16:26,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/b3e4bc9887c14aa083503ca6410698bf is 1080, key is row0001/info:/1733321786217/Put/seqid=0 2024-12-04T14:16:26,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-04T14:16:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321796253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:26,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741839_1015 (size=12509) 2024-12-04T14:16:26,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741839_1015 (size=12509) 2024-12-04T14:16:26,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/b3e4bc9887c14aa083503ca6410698bf 2024-12-04T14:16:26,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/b3e4bc9887c14aa083503ca6410698bf as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b3e4bc9887c14aa083503ca6410698bf 2024-12-04T14:16:26,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b3e4bc9887c14aa083503ca6410698bf, entries=7, sequenceid=11, filesize=12.2 K 2024-12-04T14:16:26,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 939b7947bcc0d24d524d3c093fe60a0d in 45ms, sequenceid=11, compaction requested=false 2024-12-04T14:16:26,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:26,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:27,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:28,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:29,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:30,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:31,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:32,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:33,607 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=1, created chunk count=12, reused chunk count=39, reuseRatio=76.47% 2024-12-04T14:16:33,608 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-04T14:16:33,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:34,613 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:35,613 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:36,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-04T14:16:36,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/d04ce6b7951e4de2adfb25c899844f61 is 1080, key is row0008/info:/1733321786226/Put/seqid=0 2024-12-04T14:16:36,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741840_1016 (size=29761) 2024-12-04T14:16:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741840_1016 (size=29761) 2024-12-04T14:16:36,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/d04ce6b7951e4de2adfb25c899844f61 2024-12-04T14:16:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/d04ce6b7951e4de2adfb25c899844f61 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61 2024-12-04T14:16:36,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61, entries=23, sequenceid=37, filesize=29.1 K 2024-12-04T14:16:36,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 939b7947bcc0d24d524d3c093fe60a0d in 26ms, sequenceid=37, compaction requested=false 2024-12-04T14:16:36,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:36,375 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=41.3 K, sizeToCheck=16.0 K 2024-12-04T14:16:36,375 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:36,375 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61 because midkey is the same as first or last row 2024-12-04T14:16:36,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:37,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:38,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:16:38,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/255fb0c436a24b888e29a6c31aa043fc is 1080, key is row0031/info:/1733321796349/Put/seqid=0 2024-12-04T14:16:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741841_1017 (size=12509) 2024-12-04T14:16:38,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741841_1017 (size=12509) 2024-12-04T14:16:38,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/255fb0c436a24b888e29a6c31aa043fc 2024-12-04T14:16:38,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/255fb0c436a24b888e29a6c31aa043fc as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/255fb0c436a24b888e29a6c31aa043fc 2024-12-04T14:16:38,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/255fb0c436a24b888e29a6c31aa043fc, entries=7, sequenceid=47, filesize=12.2 K 2024-12-04T14:16:38,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for 939b7947bcc0d24d524d3c093fe60a0d in 22ms, sequenceid=47, compaction requested=true 2024-12-04T14:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=53.5 K, sizeToCheck=16.0 K 2024-12-04T14:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61 because midkey is the same as first or last row 2024-12-04T14:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 939b7947bcc0d24d524d3c093fe60a0d:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:38,380 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:16:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:38,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB 2024-12-04T14:16:38,382 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:16:38,382 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): 939b7947bcc0d24d524d3c093fe60a0d/info is initiating minor compaction (all files) 2024-12-04T14:16:38,382 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 939b7947bcc0d24d524d3c093fe60a0d/info in TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:38,382 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b3e4bc9887c14aa083503ca6410698bf, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/255fb0c436a24b888e29a6c31aa043fc] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp, totalSize=53.5 K 2024-12-04T14:16:38,382 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting b3e4bc9887c14aa083503ca6410698bf, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733321786217 2024-12-04T14:16:38,383 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting d04ce6b7951e4de2adfb25c899844f61, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733321786226 2024-12-04T14:16:38,383 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 255fb0c436a24b888e29a6c31aa043fc, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733321796349 2024-12-04T14:16:38,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/fb8aeb919bae45bbb36fac25f9925181 is 1080, key is row0038/info:/1733321798358/Put/seqid=0 2024-12-04T14:16:38,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741842_1018 (size=27607) 2024-12-04T14:16:38,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741842_1018 (size=27607) 2024-12-04T14:16:38,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/fb8aeb919bae45bbb36fac25f9925181 2024-12-04T14:16:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/fb8aeb919bae45bbb36fac25f9925181 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/fb8aeb919bae45bbb36fac25f9925181 2024-12-04T14:16:38,403 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 939b7947bcc0d24d524d3c093fe60a0d#info#compaction#45 average throughput is 9.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:16:38,404 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/257815d46e0e4d4a86c6be0be57cdfa5 is 1080, key is row0001/info:/1733321786217/Put/seqid=0 2024-12-04T14:16:38,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741843_1019 (size=44978) 2024-12-04T14:16:38,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/fb8aeb919bae45bbb36fac25f9925181, entries=21, sequenceid=71, filesize=27.0 K 2024-12-04T14:16:38,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741843_1019 (size=44978) 2024-12-04T14:16:38,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=6.30 KB/6456 for 939b7947bcc0d24d524d3c093fe60a0d in 28ms, sequenceid=71, compaction requested=false 2024-12-04T14:16:38,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:38,410 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=80.5 K, sizeToCheck=16.0 K 2024-12-04T14:16:38,410 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:38,410 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61 because midkey is the same as first or last row 2024-12-04T14:16:38,414 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/257815d46e0e4d4a86c6be0be57cdfa5 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5 2024-12-04T14:16:38,420 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 939b7947bcc0d24d524d3c093fe60a0d/info of 939b7947bcc0d24d524d3c093fe60a0d into 257815d46e0e4d4a86c6be0be57cdfa5(size=43.9 K), total size for store is 70.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:16:38,420 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:38,420 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d., storeName=939b7947bcc0d24d524d3c093fe60a0d/info, priority=13, startTime=1733321798380; duration=0sec 2024-12-04T14:16:38,420 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=70.9 K, sizeToCheck=16.0 K 2024-12-04T14:16:38,420 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:38,420 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5 because midkey is the same as first or last row 2024-12-04T14:16:38,420 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:38,420 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 939b7947bcc0d24d524d3c093fe60a0d:info 2024-12-04T14:16:38,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:39,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:40,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,174 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,174 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,174 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,174 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,195 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:16:40,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/b49e2111961c43b8864f7233f76b2e36 is 1080, key is row0059/info:/1733321798381/Put/seqid=0 2024-12-04T14:16:40,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741844_1020 (size=12509) 2024-12-04T14:16:40,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741844_1020 (size=12509) 2024-12-04T14:16:40,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/b49e2111961c43b8864f7233f76b2e36 2024-12-04T14:16:40,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/b49e2111961c43b8864f7233f76b2e36 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b49e2111961c43b8864f7233f76b2e36 2024-12-04T14:16:40,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b49e2111961c43b8864f7233f76b2e36, entries=7, sequenceid=82, filesize=12.2 K 2024-12-04T14:16:40,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for 939b7947bcc0d24d524d3c093fe60a0d in 24ms, sequenceid=82, compaction requested=true 2024-12-04T14:16:40,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=83.1 K, sizeToCheck=16.0 K 2024-12-04T14:16:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5 because midkey is the same as first or last row 2024-12-04T14:16:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 939b7947bcc0d24d524d3c093fe60a0d:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:16:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:40,415 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:16:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=23.12 KB heapSize=25 KB 2024-12-04T14:16:40,416 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85094 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:16:40,416 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): 939b7947bcc0d24d524d3c093fe60a0d/info is initiating minor compaction (all files) 2024-12-04T14:16:40,416 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 939b7947bcc0d24d524d3c093fe60a0d/info in TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:40,416 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/fb8aeb919bae45bbb36fac25f9925181, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b49e2111961c43b8864f7233f76b2e36] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp, totalSize=83.1 K 2024-12-04T14:16:40,417 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 257815d46e0e4d4a86c6be0be57cdfa5, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733321786217 2024-12-04T14:16:40,417 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting fb8aeb919bae45bbb36fac25f9925181, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733321798358 2024-12-04T14:16:40,418 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting b49e2111961c43b8864f7233f76b2e36, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733321798381 2024-12-04T14:16:40,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/f8e00b308f014a98906d925e2ddc1ade is 1080, key is row0066/info:/1733321800391/Put/seqid=0 2024-12-04T14:16:40,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-04T14:16:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321810425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741845_1021 (size=28684) 2024-12-04T14:16:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741845_1021 (size=28684) 2024-12-04T14:16:40,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/f8e00b308f014a98906d925e2ddc1ade 2024-12-04T14:16:40,434 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 939b7947bcc0d24d524d3c093fe60a0d#info#compaction#48 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:16:40,435 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/de347c398e5345d7a7d03eaad2ec0198 is 1080, key is row0001/info:/1733321786217/Put/seqid=0 2024-12-04T14:16:40,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741846_1022 (size=75378) 2024-12-04T14:16:40,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741846_1022 (size=75378) 2024-12-04T14:16:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/f8e00b308f014a98906d925e2ddc1ade as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/f8e00b308f014a98906d925e2ddc1ade 2024-12-04T14:16:40,445 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/de347c398e5345d7a7d03eaad2ec0198 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198 2024-12-04T14:16:40,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/f8e00b308f014a98906d925e2ddc1ade, entries=22, sequenceid=107, filesize=28.0 K 2024-12-04T14:16:40,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=7.36 KB/7532 for 939b7947bcc0d24d524d3c093fe60a0d in 31ms, sequenceid=107, compaction requested=false 2024-12-04T14:16:40,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:40,446 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=111.1 K, sizeToCheck=16.0 K 2024-12-04T14:16:40,446 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:40,446 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5 because midkey is the same as first or last row 2024-12-04T14:16:40,450 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 939b7947bcc0d24d524d3c093fe60a0d/info of 939b7947bcc0d24d524d3c093fe60a0d into de347c398e5345d7a7d03eaad2ec0198(size=73.6 K), total size for store is 101.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:16:40,450 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:40,450 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d., storeName=939b7947bcc0d24d524d3c093fe60a0d/info, priority=13, startTime=1733321800415; duration=0sec 2024-12-04T14:16:40,450 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=101.6 K, sizeToCheck=16.0 K 2024-12-04T14:16:40,450 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T14:16:40,451 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:40,451 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:40,451 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 939b7947bcc0d24d524d3c093fe60a0d:info 2024-12-04T14:16:40,452 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40263 {}] assignment.AssignmentManager(1346): Split request from ec15ac6edd88,37995,1733321774863, parent={ENCODED => 939b7947bcc0d24d524d3c093fe60a0d, NAME => 'TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-12-04T14:16:40,457 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40263 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:40,460 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40263 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=939b7947bcc0d24d524d3c093fe60a0d, daughterA=e69ba94e2d59fa312508f78c59f9c076, daughterB=fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:40,462 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=939b7947bcc0d24d524d3c093fe60a0d, daughterA=e69ba94e2d59fa312508f78c59f9c076, daughterB=fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:40,462 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=939b7947bcc0d24d524d3c093fe60a0d, daughterA=e69ba94e2d59fa312508f78c59f9c076, daughterB=fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:40,462 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=939b7947bcc0d24d524d3c093fe60a0d, daughterA=e69ba94e2d59fa312508f78c59f9c076, daughterB=fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:40,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, UNASSIGN}] 2024-12-04T14:16:40,469 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, UNASSIGN 2024-12-04T14:16:40,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=939b7947bcc0d24d524d3c093fe60a0d, regionState=CLOSING, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:40,471 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-12-04T14:16:40,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; CloseRegionProcedure 939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863}] 2024-12-04T14:16:40,616 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:40,626 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:40,627 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(124): Close 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,627 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(138): Unassign region: split region: true: evictCache: true 2024-12-04T14:16:40,628 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1681): Closing 939b7947bcc0d24d524d3c093fe60a0d, disabling compactions & flushes 2024-12-04T14:16:40,628 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:40,628 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:40,628 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. after waiting 0 ms 2024-12-04T14:16:40,628 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:40,628 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(2837): Flushing 939b7947bcc0d24d524d3c093fe60a0d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:16:40,633 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/6a068697363048e1be7840a49daf4820 is 1080, key is row0088/info:/1733321800416/Put/seqid=0 2024-12-04T14:16:40,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741847_1023 (size=12509) 2024-12-04T14:16:40,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741847_1023 (size=12509) 2024-12-04T14:16:40,639 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/6a068697363048e1be7840a49daf4820 2024-12-04T14:16:40,645 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/.tmp/info/6a068697363048e1be7840a49daf4820 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/6a068697363048e1be7840a49daf4820 2024-12-04T14:16:40,650 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/6a068697363048e1be7840a49daf4820, entries=7, sequenceid=118, filesize=12.2 K 2024-12-04T14:16:40,651 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 939b7947bcc0d24d524d3c093fe60a0d in 23ms, sequenceid=118, compaction requested=true 2024-12-04T14:16:40,652 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b3e4bc9887c14aa083503ca6410698bf, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/255fb0c436a24b888e29a6c31aa043fc, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/fb8aeb919bae45bbb36fac25f9925181, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b49e2111961c43b8864f7233f76b2e36] to archive 2024-12-04T14:16:40,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:16:40,655 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b3e4bc9887c14aa083503ca6410698bf to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b3e4bc9887c14aa083503ca6410698bf 2024-12-04T14:16:40,656 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/d04ce6b7951e4de2adfb25c899844f61 2024-12-04T14:16:40,657 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/257815d46e0e4d4a86c6be0be57cdfa5 2024-12-04T14:16:40,658 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/255fb0c436a24b888e29a6c31aa043fc to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/255fb0c436a24b888e29a6c31aa043fc 2024-12-04T14:16:40,660 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/fb8aeb919bae45bbb36fac25f9925181 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/fb8aeb919bae45bbb36fac25f9925181 2024-12-04T14:16:40,661 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b49e2111961c43b8864f7233f76b2e36 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/b49e2111961c43b8864f7233f76b2e36 2024-12-04T14:16:40,665 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=1 2024-12-04T14:16:40,666 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. 2024-12-04T14:16:40,666 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1635): Region close journal for 939b7947bcc0d24d524d3c093fe60a0d: 2024-12-04T14:16:40,668 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(170): Closed 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=939b7947bcc0d24d524d3c093fe60a0d, regionState=CLOSED 2024-12-04T14:16:40,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T14:16:40,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; CloseRegionProcedure 939b7947bcc0d24d524d3c093fe60a0d, server=ec15ac6edd88,37995,1733321774863 in 199 msec 2024-12-04T14:16:40,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-04T14:16:40,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=939b7947bcc0d24d524d3c093fe60a0d, UNASSIGN in 204 msec 2024-12-04T14:16:40,695 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:40,697 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=12 splitting 3 storefiles, region=939b7947bcc0d24d524d3c093fe60a0d, threads=3 2024-12-04T14:16:40,698 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/6a068697363048e1be7840a49daf4820 for region: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,698 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198 for region: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,698 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/f8e00b308f014a98906d925e2ddc1ade for region: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,701 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:16:40,703 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,703 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,703 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,704 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,704 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,704 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,708 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/6a068697363048e1be7840a49daf4820, top=true 2024-12-04T14:16:40,718 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/f8e00b308f014a98906d925e2ddc1ade, top=true 2024-12-04T14:16:40,725 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,726 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,726 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,726 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,726 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,727 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,728 INFO [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820 for child: fda0efb5cff38b91cf5f681856c2db39, parent: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,728 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/6a068697363048e1be7840a49daf4820 for region: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,730 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,730 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,731 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,732 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade for child: fda0efb5cff38b91cf5f681856c2db39, parent: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,732 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/f8e00b308f014a98906d925e2ddc1ade for region: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741848_1024 (size=27) 2024-12-04T14:16:40,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741848_1024 (size=27) 2024-12-04T14:16:40,734 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:40,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741849_1025 (size=27) 2024-12-04T14:16:40,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741849_1025 (size=27) 2024-12-04T14:16:40,746 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198 for region: 939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:16:40,747 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=12 split storefiles for region 939b7947bcc0d24d524d3c093fe60a0d Daughter A: [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d] storefiles, Daughter B: [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade] storefiles. 2024-12-04T14:16:40,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741850_1026 (size=71) 2024-12-04T14:16:40,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741850_1026 (size=71) 2024-12-04T14:16:40,756 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:40,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741851_1027 (size=71) 2024-12-04T14:16:40,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741851_1027 (size=71) 2024-12-04T14:16:40,770 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:40,779 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1 2024-12-04T14:16:40,781 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1 2024-12-04T14:16:40,783 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733321800782"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733321800782"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733321800782"}]},"ts":"1733321800782"} 2024-12-04T14:16:40,783 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733321800782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321800782"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733321800782"}]},"ts":"1733321800782"} 2024-12-04T14:16:40,783 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733321800782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321800782"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733321800782"}]},"ts":"1733321800782"} 2024-12-04T14:16:40,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37995 {}] regionserver.HRegion(8581): Flush requested on 1588230740 2024-12-04T14:16:40,815 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:16:40,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=4.75 KB heapSize=8.29 KB 2024-12-04T14:16:40,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e69ba94e2d59fa312508f78c59f9c076, ASSIGN}, {pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fda0efb5cff38b91cf5f681856c2db39, ASSIGN}] 2024-12-04T14:16:40,820 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e69ba94e2d59fa312508f78c59f9c076, ASSIGN 2024-12-04T14:16:40,820 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fda0efb5cff38b91cf5f681856c2db39, ASSIGN 2024-12-04T14:16:40,821 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e69ba94e2d59fa312508f78c59f9c076, ASSIGN; state=SPLITTING_NEW, location=ec15ac6edd88,37995,1733321774863; forceNewPlan=false, retain=false 2024-12-04T14:16:40,821 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fda0efb5cff38b91cf5f681856c2db39, ASSIGN; state=SPLITTING_NEW, location=ec15ac6edd88,37995,1733321774863; forceNewPlan=false, retain=false 2024-12-04T14:16:40,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/info/0fbefd2e289e4e9c83328a05fed00b6e is 193, key is TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39./info:regioninfo/1733321800782/Put/seqid=0 2024-12-04T14:16:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741852_1028 (size=9423) 2024-12-04T14:16:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741852_1028 (size=9423) 2024-12-04T14:16:40,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.54 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/info/0fbefd2e289e4e9c83328a05fed00b6e 2024-12-04T14:16:40,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/table/d3c91dcc53af4e5497ae91eb3e65b6c3 is 65, key is TestLogRolling-testLogRolling/table:state/1733321776566/Put/seqid=0 2024-12-04T14:16:40,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741853_1029 (size=5412) 2024-12-04T14:16:40,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741853_1029 (size=5412) 2024-12-04T14:16:40,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=216 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/table/d3c91dcc53af4e5497ae91eb3e65b6c3 2024-12-04T14:16:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/info/0fbefd2e289e4e9c83328a05fed00b6e as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/info/0fbefd2e289e4e9c83328a05fed00b6e 2024-12-04T14:16:40,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/info/0fbefd2e289e4e9c83328a05fed00b6e, entries=29, sequenceid=17, filesize=9.2 K 2024-12-04T14:16:40,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/table/d3c91dcc53af4e5497ae91eb3e65b6c3 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/table/d3c91dcc53af4e5497ae91eb3e65b6c3 2024-12-04T14:16:40,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/table/d3c91dcc53af4e5497ae91eb3e65b6c3, entries=4, sequenceid=17, filesize=5.3 K 2024-12-04T14:16:40,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.75 KB/4869, heapSize ~8.01 KB/8200, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=17, compaction requested=false 2024-12-04T14:16:40,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-04T14:16:40,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=e69ba94e2d59fa312508f78c59f9c076, regionState=OPENING, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:40,971 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=fda0efb5cff38b91cf5f681856c2db39, regionState=OPENING, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:40,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863}] 2024-12-04T14:16:40,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=15, state=RUNNABLE; OpenRegionProcedure e69ba94e2d59fa312508f78c59f9c076, server=ec15ac6edd88,37995,1733321774863}] 2024-12-04T14:16:41,125 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:41,128 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:16:41,128 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => fda0efb5cff38b91cf5f681856c2db39, NAME => 'TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.', STARTKEY => 'row0062', ENDKEY => ''} 2024-12-04T14:16:41,129 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,129 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:41,129 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,129 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,130 INFO [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,131 INFO [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fda0efb5cff38b91cf5f681856c2db39 columnFamilyName info 2024-12-04T14:16:41,131 DEBUG [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:41,140 DEBUG [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820 2024-12-04T14:16:41,143 DEBUG [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade 2024-12-04T14:16:41,150 DEBUG [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d->hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198-top 2024-12-04T14:16:41,151 INFO [StoreOpener-fda0efb5cff38b91cf5f681856c2db39-1 {}] regionserver.HStore(327): Store=fda0efb5cff38b91cf5f681856c2db39/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:41,151 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,152 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,154 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:16:41,155 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1102): Opened fda0efb5cff38b91cf5f681856c2db39; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=737494, jitterRate=-0.06222797930240631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:16:41,156 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:16:41,157 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., pid=17, masterSystemTime=1733321801125 2024-12-04T14:16:41,157 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:16:41,157 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:41,157 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:16:41,158 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:16:41,158 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:16:41,158 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:16:41,158 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d->hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198-top, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=113.8 K 2024-12-04T14:16:41,158 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:16:41,158 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:16:41,158 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:16:41,159 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7285): Opening region: {ENCODED => e69ba94e2d59fa312508f78c59f9c076, NAME => 'TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.', STARTKEY => '', ENDKEY => 'row0062'} 2024-12-04T14:16:41,159 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733321786217 2024-12-04T14:16:41,159 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,159 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:16:41,159 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7327): checking encryption for e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,159 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7330): checking classloading for e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=fda0efb5cff38b91cf5f681856c2db39, regionState=OPEN, openSeqNum=122, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:41,159 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733321800391 2024-12-04T14:16:41,160 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733321800416 2024-12-04T14:16:41,160 INFO [StoreOpener-e69ba94e2d59fa312508f78c59f9c076-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,161 INFO [StoreOpener-e69ba94e2d59fa312508f78c59f9c076-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e69ba94e2d59fa312508f78c59f9c076 columnFamilyName info 2024-12-04T14:16:41,161 DEBUG [StoreOpener-e69ba94e2d59fa312508f78c59f9c076-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:16:41,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T14:16:41,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 in 188 msec 2024-12-04T14:16:41,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fda0efb5cff38b91cf5f681856c2db39, ASSIGN in 344 msec 2024-12-04T14:16:41,171 DEBUG [StoreOpener-e69ba94e2d59fa312508f78c59f9c076-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d->hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198-bottom 2024-12-04T14:16:41,172 INFO [StoreOpener-e69ba94e2d59fa312508f78c59f9c076-1 {}] regionserver.HStore(327): Store=e69ba94e2d59fa312508f78c59f9c076/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:16:41,173 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,173 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,175 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1085): writing seq id for e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:16:41,176 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1102): Opened e69ba94e2d59fa312508f78c59f9c076; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=779155, jitterRate=-0.009253770112991333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:16:41,176 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1001): Region open journal for e69ba94e2d59fa312508f78c59f9c076: 2024-12-04T14:16:41,177 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076., pid=18, masterSystemTime=1733321801125 2024-12-04T14:16:41,177 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(403): Add compact mark for store e69ba94e2d59fa312508f78c59f9c076:info, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:16:41,177 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:41,177 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T14:16:41,177 INFO [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:16:41,177 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HStore(1540): e69ba94e2d59fa312508f78c59f9c076/info is initiating minor compaction (all files) 2024-12-04T14:16:41,178 INFO [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e69ba94e2d59fa312508f78c59f9c076/info in TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:16:41,178 INFO [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d->hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198-bottom] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/.tmp, totalSize=73.6 K 2024-12-04T14:16:41,178 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] compactions.Compactor(224): Compacting de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733321786217 2024-12-04T14:16:41,179 DEBUG [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:16:41,179 INFO [RS_OPEN_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:16:41,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=e69ba94e2d59fa312508f78c59f9c076, regionState=OPEN, openSeqNum=122, regionLocation=ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:41,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=15 2024-12-04T14:16:41,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=15, state=SUCCESS; OpenRegionProcedure e69ba94e2d59fa312508f78c59f9c076, server=ec15ac6edd88,37995,1733321774863 in 207 msec 2024-12-04T14:16:41,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=12 2024-12-04T14:16:41,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e69ba94e2d59fa312508f78c59f9c076, ASSIGN in 363 msec 2024-12-04T14:16:41,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=939b7947bcc0d24d524d3c093fe60a0d, daughterA=e69ba94e2d59fa312508f78c59f9c076, daughterB=fda0efb5cff38b91cf5f681856c2db39 in 728 msec 2024-12-04T14:16:41,192 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#52 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:16:41,193 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1226b7f861374e8ebd9897775f48c5f1 is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:16:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741854_1030 (size=40830) 2024-12-04T14:16:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741854_1030 (size=40830) 2024-12-04T14:16:41,207 INFO [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e69ba94e2d59fa312508f78c59f9c076#info#compaction#53 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:16:41,207 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/.tmp/info/7ce6607800a546b4b5fd1405b8874888 is 1080, key is row0001/info:/1733321786217/Put/seqid=0 2024-12-04T14:16:41,209 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1226b7f861374e8ebd9897775f48c5f1 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1226b7f861374e8ebd9897775f48c5f1 2024-12-04T14:16:41,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741855_1031 (size=70862) 2024-12-04T14:16:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741855_1031 (size=70862) 2024-12-04T14:16:41,216 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into 1226b7f861374e8ebd9897775f48c5f1(size=39.9 K), total size for store is 39.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:16:41,216 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:16:41,217 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321801157; duration=0sec 2024-12-04T14:16:41,217 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:41,217 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:16:41,219 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/.tmp/info/7ce6607800a546b4b5fd1405b8874888 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/7ce6607800a546b4b5fd1405b8874888 2024-12-04T14:16:41,225 INFO [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 1 (all) file(s) in e69ba94e2d59fa312508f78c59f9c076/info of e69ba94e2d59fa312508f78c59f9c076 into 7ce6607800a546b4b5fd1405b8874888(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:16:41,225 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e69ba94e2d59fa312508f78c59f9c076: 2024-12-04T14:16:41,225 INFO [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076., storeName=e69ba94e2d59fa312508f78c59f9c076/info, priority=15, startTime=1733321801177; duration=0sec 2024-12-04T14:16:41,225 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:16:41,225 DEBUG [RS:0;ec15ac6edd88:37995-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e69ba94e2d59fa312508f78c59f9c076:info 2024-12-04T14:16:41,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:42,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:43,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:44,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:44,791 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:16:45,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:46,168 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T14:16:46,169 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,169 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,188 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,188 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,188 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,189 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,190 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,195 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,195 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,195 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,199 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T14:16:46,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:47,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:48,107 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:16:48,109 INFO [RS-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43730, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:16:48,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:49,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321820484, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733321776207.939b7947bcc0d24d524d3c093fe60a0d. is not online on ec15ac6edd88,37995,1733321774863 2024-12-04T14:16:50,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:51,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:52,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:53,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:54,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:55,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:56,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:56,978 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:16:56,978 INFO [master/ec15ac6edd88:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:16:57,626 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:58,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:16:59,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:00,624 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835 2024-12-04T14:17:00,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:01,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:02,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:03,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:04,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:05,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:06,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:07,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:08,632 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:09,633 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:10,633 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:11,634 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:12,634 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:12,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:17:12,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/6ac93f48bc8d46aa9990e6d670d448a8 is 1080, key is row0095/info:/1733321830688/Put/seqid=0 2024-12-04T14:17:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741856_1032 (size=12513) 2024-12-04T14:17:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741856_1032 (size=12513) 2024-12-04T14:17:12,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/6ac93f48bc8d46aa9990e6d670d448a8 2024-12-04T14:17:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/6ac93f48bc8d46aa9990e6d670d448a8 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6ac93f48bc8d46aa9990e6d670d448a8 2024-12-04T14:17:12,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-04T14:17:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321842720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 2024-12-04T14:17:12,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6ac93f48bc8d46aa9990e6d670d448a8, entries=7, sequenceid=132, filesize=12.2 K 2024-12-04T14:17:12,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for fda0efb5cff38b91cf5f681856c2db39 in 25ms, sequenceid=132, compaction requested=false 2024-12-04T14:17:12,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:13,635 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:14,635 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:14,791 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:17:15,636 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:15,661 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T14:17:15,661 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 8dc4e070b920b05210aa2083cb985207 changed from -1.0 to 0.0, refreshing cache 2024-12-04T14:17:16,637 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:17,637 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:18,638 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:19,638 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:20,639 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:21,639 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:22,640 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:22,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-04T14:17:22,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/b7c9395f25e845e28cb35ec4f49544e0 is 1080, key is row0102/info:/1733321832697/Put/seqid=0 2024-12-04T14:17:22,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741857_1033 (size=29784) 2024-12-04T14:17:22,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741857_1033 (size=29784) 2024-12-04T14:17:22,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/b7c9395f25e845e28cb35ec4f49544e0 2024-12-04T14:17:22,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/b7c9395f25e845e28cb35ec4f49544e0 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/b7c9395f25e845e28cb35ec4f49544e0 2024-12-04T14:17:22,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/b7c9395f25e845e28cb35ec4f49544e0, entries=23, sequenceid=158, filesize=29.1 K 2024-12-04T14:17:22,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for fda0efb5cff38b91cf5f681856c2db39 in 22ms, sequenceid=158, compaction requested=true 2024-12-04T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:22,795 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:17:22,796 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 83127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:17:22,796 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:17:22,796 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:17:22,797 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1226b7f861374e8ebd9897775f48c5f1, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6ac93f48bc8d46aa9990e6d670d448a8, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/b7c9395f25e845e28cb35ec4f49544e0] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=81.2 K 2024-12-04T14:17:22,797 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1226b7f861374e8ebd9897775f48c5f1, keycount=33, bloomtype=ROW, size=39.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733321798385 2024-12-04T14:17:22,797 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6ac93f48bc8d46aa9990e6d670d448a8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733321830688 2024-12-04T14:17:22,798 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting b7c9395f25e845e28cb35ec4f49544e0, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733321832697 2024-12-04T14:17:22,808 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#56 average throughput is 64.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:17:22,809 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/eee0cbc4a66a43e1bf64a1630a49a3ee is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:17:22,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741858_1034 (size=73410) 2024-12-04T14:17:22,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741858_1034 (size=73410) 2024-12-04T14:17:22,818 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/eee0cbc4a66a43e1bf64a1630a49a3ee as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/eee0cbc4a66a43e1bf64a1630a49a3ee 2024-12-04T14:17:22,824 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into eee0cbc4a66a43e1bf64a1630a49a3ee(size=71.7 K), total size for store is 71.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:17:22,824 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:22,824 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321842795; duration=0sec 2024-12-04T14:17:22,824 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:22,824 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:17:23,641 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:24,641 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:24,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:17:24,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/712bfda33f8d416aae8eb0051e426092 is 1080, key is row0125/info:/1733321842774/Put/seqid=0 2024-12-04T14:17:24,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741859_1035 (size=12516) 2024-12-04T14:17:24,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741859_1035 (size=12516) 2024-12-04T14:17:24,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/712bfda33f8d416aae8eb0051e426092 2024-12-04T14:17:24,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/712bfda33f8d416aae8eb0051e426092 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/712bfda33f8d416aae8eb0051e426092 2024-12-04T14:17:24,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/712bfda33f8d416aae8eb0051e426092, entries=7, sequenceid=169, filesize=12.2 K 2024-12-04T14:17:24,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=22.07 KB/22596 for fda0efb5cff38b91cf5f681856c2db39 in 24ms, sequenceid=169, compaction requested=false 2024-12-04T14:17:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:24,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-04T14:17:24,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1cd1c0e81ec9418f9aecf49efb75dd6b is 1080, key is row0132/info:/1733321844783/Put/seqid=0 2024-12-04T14:17:24,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741860_1036 (size=29784) 2024-12-04T14:17:24,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741860_1036 (size=29784) 2024-12-04T14:17:24,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1cd1c0e81ec9418f9aecf49efb75dd6b 2024-12-04T14:17:24,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1cd1c0e81ec9418f9aecf49efb75dd6b as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1cd1c0e81ec9418f9aecf49efb75dd6b 2024-12-04T14:17:24,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1cd1c0e81ec9418f9aecf49efb75dd6b, entries=23, sequenceid=195, filesize=29.1 K 2024-12-04T14:17:24,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for fda0efb5cff38b91cf5f681856c2db39 in 20ms, sequenceid=195, compaction requested=true 2024-12-04T14:17:24,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:24,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:17:24,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:24,827 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:17:24,829 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 115710 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:17:24,829 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:17:24,829 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:17:24,829 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/eee0cbc4a66a43e1bf64a1630a49a3ee, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/712bfda33f8d416aae8eb0051e426092, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1cd1c0e81ec9418f9aecf49efb75dd6b] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=113.0 K 2024-12-04T14:17:24,829 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting eee0cbc4a66a43e1bf64a1630a49a3ee, keycount=63, bloomtype=ROW, size=71.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733321798385 2024-12-04T14:17:24,830 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 712bfda33f8d416aae8eb0051e426092, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733321842774 2024-12-04T14:17:24,830 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1cd1c0e81ec9418f9aecf49efb75dd6b, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733321844783 2024-12-04T14:17:24,844 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#59 average throughput is 31.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:17:24,845 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/dfb3f437335f4cc9a295e9f50e39258e is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:17:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741861_1037 (size=105860) 2024-12-04T14:17:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741861_1037 (size=105860) 2024-12-04T14:17:24,860 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/dfb3f437335f4cc9a295e9f50e39258e as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/dfb3f437335f4cc9a295e9f50e39258e 2024-12-04T14:17:24,865 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into dfb3f437335f4cc9a295e9f50e39258e(size=103.4 K), total size for store is 103.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:17:24,865 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:24,866 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321844827; duration=0sec 2024-12-04T14:17:24,866 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:24,866 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:17:25,642 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:26,129 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region fda0efb5cff38b91cf5f681856c2db39, had cached 0 bytes from a total of 105860 2024-12-04T14:17:26,159 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region e69ba94e2d59fa312508f78c59f9c076, had cached 0 bytes from a total of 70862 2024-12-04T14:17:26,642 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:26,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:17:26,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/0829a400952243ee9a733aedb6ed2b5c is 1080, key is row0155/info:/1733321844807/Put/seqid=0 2024-12-04T14:17:26,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741862_1038 (size=12516) 2024-12-04T14:17:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741862_1038 (size=12516) 2024-12-04T14:17:26,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/0829a400952243ee9a733aedb6ed2b5c 2024-12-04T14:17:26,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/0829a400952243ee9a733aedb6ed2b5c as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0829a400952243ee9a733aedb6ed2b5c 2024-12-04T14:17:26,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0829a400952243ee9a733aedb6ed2b5c, entries=7, sequenceid=206, filesize=12.2 K 2024-12-04T14:17:26,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-04T14:17:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321856842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 2024-12-04T14:17:26,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for fda0efb5cff38b91cf5f681856c2db39 in 27ms, sequenceid=206, compaction requested=false 2024-12-04T14:17:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:27,643 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:28,643 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:29,644 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:29,644 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta after 196116ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor238.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:17:30,645 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:31,645 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:32,646 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:33,646 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:34,647 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:35,647 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:36,648 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:36,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-04T14:17:36,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/4132a4da43fb490ead8885f87cd63196 is 1080, key is row0162/info:/1733321846816/Put/seqid=0 2024-12-04T14:17:36,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-04T14:17:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321866901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 2024-12-04T14:17:36,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741863_1039 (size=29784) 2024-12-04T14:17:36,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741863_1039 (size=29784) 2024-12-04T14:17:36,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/4132a4da43fb490ead8885f87cd63196 2024-12-04T14:17:36,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/4132a4da43fb490ead8885f87cd63196 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4132a4da43fb490ead8885f87cd63196 2024-12-04T14:17:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4132a4da43fb490ead8885f87cd63196, entries=23, sequenceid=232, filesize=29.1 K 2024-12-04T14:17:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for fda0efb5cff38b91cf5f681856c2db39 in 21ms, sequenceid=232, compaction requested=true 2024-12-04T14:17:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:17:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:36,916 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:17:36,917 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 148160 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:17:36,917 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:17:36,917 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:17:36,917 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/dfb3f437335f4cc9a295e9f50e39258e, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0829a400952243ee9a733aedb6ed2b5c, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4132a4da43fb490ead8885f87cd63196] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=144.7 K 2024-12-04T14:17:36,918 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting dfb3f437335f4cc9a295e9f50e39258e, keycount=93, bloomtype=ROW, size=103.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733321798385 2024-12-04T14:17:36,918 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0829a400952243ee9a733aedb6ed2b5c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733321844807 2024-12-04T14:17:36,918 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4132a4da43fb490ead8885f87cd63196, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733321846816 2024-12-04T14:17:36,954 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#62 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:17:36,954 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/792c45fa7307437883034cdebebf7e50 is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:17:36,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741864_1040 (size=138507) 2024-12-04T14:17:36,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741864_1040 (size=138507) 2024-12-04T14:17:36,963 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/792c45fa7307437883034cdebebf7e50 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/792c45fa7307437883034cdebebf7e50 2024-12-04T14:17:36,969 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into 792c45fa7307437883034cdebebf7e50(size=135.3 K), total size for store is 135.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:17:36,969 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:36,969 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321856916; duration=0sec 2024-12-04T14:17:36,969 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:36,970 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:17:37,649 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:38,649 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:39,650 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:40,650 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:41,651 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:42,651 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:43,652 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:44,652 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:44,792 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:17:45,624 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835 2024-12-04T14:17:45,653 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:46,653 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:46,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:17:46,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/7a7b5159767a419fa9d430dd73566f4f is 1080, key is row0185/info:/1733321856895/Put/seqid=0 2024-12-04T14:17:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741865_1041 (size=12516) 2024-12-04T14:17:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741865_1041 (size=12516) 2024-12-04T14:17:46,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/7a7b5159767a419fa9d430dd73566f4f 2024-12-04T14:17:46,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/7a7b5159767a419fa9d430dd73566f4f as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7a7b5159767a419fa9d430dd73566f4f 2024-12-04T14:17:46,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7a7b5159767a419fa9d430dd73566f4f, entries=7, sequenceid=243, filesize=12.2 K 2024-12-04T14:17:46,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=1.05 KB/1076 for fda0efb5cff38b91cf5f681856c2db39 in 19ms, sequenceid=243, compaction requested=false 2024-12-04T14:17:46,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:47,654 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:48,655 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:48,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:17:48,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1a306769e18348f7b7063cb623b0af89 is 1080, key is row0192/info:/1733321866954/Put/seqid=0 2024-12-04T14:17:48,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741866_1042 (size=12516) 2024-12-04T14:17:48,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741866_1042 (size=12516) 2024-12-04T14:17:48,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1a306769e18348f7b7063cb623b0af89 2024-12-04T14:17:48,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/1a306769e18348f7b7063cb623b0af89 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1a306769e18348f7b7063cb623b0af89 2024-12-04T14:17:48,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1a306769e18348f7b7063cb623b0af89, entries=7, sequenceid=253, filesize=12.2 K 2024-12-04T14:17:48,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for fda0efb5cff38b91cf5f681856c2db39 in 21ms, sequenceid=253, compaction requested=true 2024-12-04T14:17:48,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:48,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:17:48,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:48,983 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:17:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:48,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB 2024-12-04T14:17:48,984 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 163539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:17:48,984 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:17:48,985 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:17:48,985 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/792c45fa7307437883034cdebebf7e50, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7a7b5159767a419fa9d430dd73566f4f, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1a306769e18348f7b7063cb623b0af89] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=159.7 K 2024-12-04T14:17:48,985 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 792c45fa7307437883034cdebebf7e50, keycount=123, bloomtype=ROW, size=135.3 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733321798385 2024-12-04T14:17:48,986 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7a7b5159767a419fa9d430dd73566f4f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733321856895 2024-12-04T14:17:48,986 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1a306769e18348f7b7063cb623b0af89, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733321866954 2024-12-04T14:17:48,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/7f8c54fcd2ab42b9883b751ba907e428 is 1080, key is row0199/info:/1733321868962/Put/seqid=0 2024-12-04T14:17:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741867_1043 (size=27648) 2024-12-04T14:17:48,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/7f8c54fcd2ab42b9883b751ba907e428 2024-12-04T14:17:48,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741867_1043 (size=27648) 2024-12-04T14:17:48,999 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#66 average throughput is 46.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:17:48,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/7f8c54fcd2ab42b9883b751ba907e428 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7f8c54fcd2ab42b9883b751ba907e428 2024-12-04T14:17:48,999 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/9f414277f2ec47a082755ae9b4ac5595 is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:17:49,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7f8c54fcd2ab42b9883b751ba907e428, entries=21, sequenceid=277, filesize=27 K 2024-12-04T14:17:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741868_1044 (size=153701) 2024-12-04T14:17:49,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=5.25 KB/5380 for fda0efb5cff38b91cf5f681856c2db39 in 21ms, sequenceid=277, compaction requested=false 2024-12-04T14:17:49,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741868_1044 (size=153701) 2024-12-04T14:17:49,011 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/9f414277f2ec47a082755ae9b4ac5595 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/9f414277f2ec47a082755ae9b4ac5595 2024-12-04T14:17:49,016 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into 9f414277f2ec47a082755ae9b4ac5595(size=150.1 K), total size for store is 177.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:17:49,016 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:49,016 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321868983; duration=0sec 2024-12-04T14:17:49,016 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:49,016 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:17:49,655 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:50,656 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:50,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T14:17:50,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/6291182bfdaf4e8887c71fa21e9d6206 is 1080, key is row0220/info:/1733321868984/Put/seqid=0 2024-12-04T14:17:51,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741869_1045 (size=12523) 2024-12-04T14:17:51,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741869_1045 (size=12523) 2024-12-04T14:17:51,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/6291182bfdaf4e8887c71fa21e9d6206 2024-12-04T14:17:51,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/6291182bfdaf4e8887c71fa21e9d6206 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6291182bfdaf4e8887c71fa21e9d6206 2024-12-04T14:17:51,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6291182bfdaf4e8887c71fa21e9d6206, entries=7, sequenceid=288, filesize=12.2 K 2024-12-04T14:17:51,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for fda0efb5cff38b91cf5f681856c2db39 in 21ms, sequenceid=288, compaction requested=true 2024-12-04T14:17:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:17:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:51,013 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:17:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:17:51,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=23.12 KB heapSize=25 KB 2024-12-04T14:17:51,014 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 193872 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:17:51,015 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:17:51,015 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:17:51,015 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/9f414277f2ec47a082755ae9b4ac5595, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7f8c54fcd2ab42b9883b751ba907e428, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6291182bfdaf4e8887c71fa21e9d6206] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=189.3 K 2024-12-04T14:17:51,015 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9f414277f2ec47a082755ae9b4ac5595, keycount=137, bloomtype=ROW, size=150.1 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733321798385 2024-12-04T14:17:51,016 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7f8c54fcd2ab42b9883b751ba907e428, keycount=21, bloomtype=ROW, size=27 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733321868962 2024-12-04T14:17:51,016 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6291182bfdaf4e8887c71fa21e9d6206, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733321868984 2024-12-04T14:17:51,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/4f07f383429a42e4a88b4baf325479cf is 1080, key is row0227/info:/1733321870992/Put/seqid=0 2024-12-04T14:17:51,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-04T14:17:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42268 deadline: 1733321881022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=fda0efb5cff38b91cf5f681856c2db39, server=ec15ac6edd88,37995,1733321774863 2024-12-04T14:17:51,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741870_1046 (size=28728) 2024-12-04T14:17:51,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741870_1046 (size=28728) 2024-12-04T14:17:51,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/4f07f383429a42e4a88b4baf325479cf 2024-12-04T14:17:51,034 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#69 average throughput is 56.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:17:51,035 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/0af625f6a2ed4f89841f6755bd858364 is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:17:51,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/4f07f383429a42e4a88b4baf325479cf as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4f07f383429a42e4a88b4baf325479cf 2024-12-04T14:17:51,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4f07f383429a42e4a88b4baf325479cf, entries=22, sequenceid=313, filesize=28.1 K 2024-12-04T14:17:51,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=7.36 KB/7532 for fda0efb5cff38b91cf5f681856c2db39 in 28ms, sequenceid=313, compaction requested=false 2024-12-04T14:17:51,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:51,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741871_1047 (size=184022) 2024-12-04T14:17:51,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741871_1047 (size=184022) 2024-12-04T14:17:51,051 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/0af625f6a2ed4f89841f6755bd858364 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0af625f6a2ed4f89841f6755bd858364 2024-12-04T14:17:51,058 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into 0af625f6a2ed4f89841f6755bd858364(size=179.7 K), total size for store is 207.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:17:51,058 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:17:51,058 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321871013; duration=0sec 2024-12-04T14:17:51,058 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:17:51,058 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:17:51,656 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:52,657 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:53,657 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:54,658 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:55,658 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:56,659 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:57,659 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:58,660 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:17:59,661 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:00,661 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] regionserver.HRegion(8581): Flush requested on fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:18:01,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fda0efb5cff38b91cf5f681856c2db39 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-12-04T14:18:01,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/8607be58ed9c4caaa6c5e42eff2396c4 is 1080, key is row0249/info:/1733321871014/Put/seqid=0 2024-12-04T14:18:01,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741872_1048 (size=13602) 2024-12-04T14:18:01,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741872_1048 (size=13602) 2024-12-04T14:18:01,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/8607be58ed9c4caaa6c5e42eff2396c4 2024-12-04T14:18:01,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/8607be58ed9c4caaa6c5e42eff2396c4 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/8607be58ed9c4caaa6c5e42eff2396c4 2024-12-04T14:18:01,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/8607be58ed9c4caaa6c5e42eff2396c4, entries=8, sequenceid=325, filesize=13.3 K 2024-12-04T14:18:01,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for fda0efb5cff38b91cf5f681856c2db39 in 22ms, sequenceid=325, compaction requested=true 2024-12-04T14:18:01,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:18:01,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fda0efb5cff38b91cf5f681856c2db39:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:18:01,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:18:01,121 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:18:01,122 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 226352 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:18:01,122 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1540): fda0efb5cff38b91cf5f681856c2db39/info is initiating minor compaction (all files) 2024-12-04T14:18:01,122 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fda0efb5cff38b91cf5f681856c2db39/info in TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:18:01,122 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0af625f6a2ed4f89841f6755bd858364, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4f07f383429a42e4a88b4baf325479cf, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/8607be58ed9c4caaa6c5e42eff2396c4] into tmpdir=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp, totalSize=221.0 K 2024-12-04T14:18:01,122 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0af625f6a2ed4f89841f6755bd858364, keycount=165, bloomtype=ROW, size=179.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733321798385 2024-12-04T14:18:01,123 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4f07f383429a42e4a88b4baf325479cf, keycount=22, bloomtype=ROW, size=28.1 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733321870992 2024-12-04T14:18:01,123 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8607be58ed9c4caaa6c5e42eff2396c4, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733321871014 2024-12-04T14:18:01,138 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fda0efb5cff38b91cf5f681856c2db39#info#compaction#71 average throughput is 40.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:18:01,138 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/3ae5ce63259b420f9af967bda10bd504 is 1080, key is row0062/info:/1733321798385/Put/seqid=0 2024-12-04T14:18:01,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741873_1049 (size=216571) 2024-12-04T14:18:01,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741873_1049 (size=216571) 2024-12-04T14:18:01,146 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/.tmp/info/3ae5ce63259b420f9af967bda10bd504 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/3ae5ce63259b420f9af967bda10bd504 2024-12-04T14:18:01,151 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fda0efb5cff38b91cf5f681856c2db39/info of fda0efb5cff38b91cf5f681856c2db39 into 3ae5ce63259b420f9af967bda10bd504(size=211.5 K), total size for store is 211.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:18:01,151 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:18:01,151 INFO [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., storeName=fda0efb5cff38b91cf5f681856c2db39/info, priority=13, startTime=1733321881121; duration=0sec 2024-12-04T14:18:01,151 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:18:01,151 DEBUG [RS:0;ec15ac6edd88:37995-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fda0efb5cff38b91cf5f681856c2db39:info 2024-12-04T14:18:01,662 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:02,662 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:03,099 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-12-04T14:18:03,099 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C37995%2C1733321774863.1733321883099 2024-12-04T14:18:03,106 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321775254 with entries=311, filesize=307.59 KB; new WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321883099 2024-12-04T14:18:03,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39193:39193),(127.0.0.1/127.0.0.1:37765:37765)] 2024-12-04T14:18:03,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321775254 is not closed yet, will try archiving it next time 2024-12-04T14:18:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741833_1009 (size=314980) 2024-12-04T14:18:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741833_1009 (size=314980) 2024-12-04T14:18:03,110 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 8dc4e070b920b05210aa2083cb985207 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-04T14:18:03,126 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/.tmp/info/bf9eeecf0b244939a25e6a3a5124ed90 is 45, key is default/info:d/1733321776073/Put/seqid=0 2024-12-04T14:18:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741875_1051 (size=5037) 2024-12-04T14:18:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741875_1051 (size=5037) 2024-12-04T14:18:03,131 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/.tmp/info/bf9eeecf0b244939a25e6a3a5124ed90 2024-12-04T14:18:03,135 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/.tmp/info/bf9eeecf0b244939a25e6a3a5124ed90 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/info/bf9eeecf0b244939a25e6a3a5124ed90 2024-12-04T14:18:03,139 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/info/bf9eeecf0b244939a25e6a3a5124ed90, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T14:18:03,140 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 8dc4e070b920b05210aa2083cb985207 in 30ms, sequenceid=6, compaction requested=false 2024-12-04T14:18:03,140 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 8dc4e070b920b05210aa2083cb985207: 2024-12-04T14:18:03,140 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.21 KB heapSize=4.13 KB 2024-12-04T14:18:03,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/info/41bdd56f754e43918dca9bdb8aca45a4 is 193, key is TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39./info:regioninfo/1733321801159/Put/seqid=0 2024-12-04T14:18:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741876_1052 (size=7803) 2024-12-04T14:18:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741876_1052 (size=7803) 2024-12-04T14:18:03,150 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.21 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/info/41bdd56f754e43918dca9bdb8aca45a4 2024-12-04T14:18:03,154 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/.tmp/info/41bdd56f754e43918dca9bdb8aca45a4 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/info/41bdd56f754e43918dca9bdb8aca45a4 2024-12-04T14:18:03,158 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/info/41bdd56f754e43918dca9bdb8aca45a4, entries=16, sequenceid=24, filesize=7.6 K 2024-12-04T14:18:03,159 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~2.21 KB/2260, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 19ms, sequenceid=24, compaction requested=false 2024-12-04T14:18:03,159 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-04T14:18:03,159 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:18:03,159 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for e69ba94e2d59fa312508f78c59f9c076: 2024-12-04T14:18:03,160 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C37995%2C1733321774863.1733321883160 2024-12-04T14:18:03,165 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321883099 with entries=2, filesize=609 B; new WAL /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321883160 2024-12-04T14:18:03,165 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37765:37765),(127.0.0.1/127.0.0.1:39193:39193)] 2024-12-04T14:18:03,165 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321883099 is not closed yet, will try archiving it next time 2024-12-04T14:18:03,166 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321775254 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/oldWALs/ec15ac6edd88%2C37995%2C1733321774863.1733321775254 2024-12-04T14:18:03,166 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T14:18:03,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741874_1050 (size=617) 2024-12-04T14:18:03,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741874_1050 (size=617) 2024-12-04T14:18:03,167 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863/ec15ac6edd88%2C37995%2C1733321774863.1733321883099 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/oldWALs/ec15ac6edd88%2C37995%2C1733321774863.1733321883099 2024-12-04T14:18:03,266 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-04T14:18:03,266 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-04T14:18:03,267 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7bfbd5bb to 127.0.0.1:54338 2024-12-04T14:18:03,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:03,267 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:18:03,267 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1799732026, stopped=false 2024-12-04T14:18:03,267 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ec15ac6edd88,40263,1733321774812 2024-12-04T14:18:03,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:18:03,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:18:03,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:03,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:03,269 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-04T14:18:03,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:03,269 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,37995,1733321774863' ***** 2024-12-04T14:18:03,269 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:18:03,269 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(3579): Received CLOSE for 8dc4e070b920b05210aa2083cb985207 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(3579): Received CLOSE for fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(3579): Received CLOSE for e69ba94e2d59fa312508f78c59f9c076 2024-12-04T14:18:03,269 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,37995,1733321774863 2024-12-04T14:18:03,269 DEBUG [RS:0;ec15ac6edd88:37995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:03,269 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8dc4e070b920b05210aa2083cb985207, disabling compactions & flushes 2024-12-04T14:18:03,270 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:18:03,270 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:18:03,270 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:18:03,270 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:18:03,270 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. after waiting 0 ms 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:18:03,270 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-12-04T14:18:03,270 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1603): Online Regions={8dc4e070b920b05210aa2083cb985207=hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207., 1588230740=hbase:meta,,1.1588230740, fda0efb5cff38b91cf5f681856c2db39=TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39., e69ba94e2d59fa312508f78c59f9c076=TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.} 2024-12-04T14:18:03,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:18:03,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:18:03,270 DEBUG [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 8dc4e070b920b05210aa2083cb985207, e69ba94e2d59fa312508f78c59f9c076, fda0efb5cff38b91cf5f681856c2db39 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:18:03,270 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:18:03,270 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:18:03,274 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/namespace/8dc4e070b920b05210aa2083cb985207/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T14:18:03,274 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/hbase/meta/1588230740/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2024-12-04T14:18:03,275 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8dc4e070b920b05210aa2083cb985207: 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733321775660.8dc4e070b920b05210aa2083cb985207. 2024-12-04T14:18:03,275 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing fda0efb5cff38b91cf5f681856c2db39, disabling compactions & flushes 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T14:18:03,275 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. after waiting 0 ms 2024-12-04T14:18:03,275 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:18:03,275 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d->hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198-top, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1226b7f861374e8ebd9897775f48c5f1, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6ac93f48bc8d46aa9990e6d670d448a8, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/eee0cbc4a66a43e1bf64a1630a49a3ee, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/b7c9395f25e845e28cb35ec4f49544e0, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/712bfda33f8d416aae8eb0051e426092, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/dfb3f437335f4cc9a295e9f50e39258e, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1cd1c0e81ec9418f9aecf49efb75dd6b, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0829a400952243ee9a733aedb6ed2b5c, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/792c45fa7307437883034cdebebf7e50, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4132a4da43fb490ead8885f87cd63196, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7a7b5159767a419fa9d430dd73566f4f, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/9f414277f2ec47a082755ae9b4ac5595, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1a306769e18348f7b7063cb623b0af89, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7f8c54fcd2ab42b9883b751ba907e428, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0af625f6a2ed4f89841f6755bd858364, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6291182bfdaf4e8887c71fa21e9d6206, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4f07f383429a42e4a88b4baf325479cf, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/8607be58ed9c4caaa6c5e42eff2396c4] to archive 2024-12-04T14:18:03,276 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:18:03,278 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:18:03,279 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-f8e00b308f014a98906d925e2ddc1ade 2024-12-04T14:18:03,280 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1226b7f861374e8ebd9897775f48c5f1 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1226b7f861374e8ebd9897775f48c5f1 2024-12-04T14:18:03,281 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/TestLogRolling-testLogRolling=939b7947bcc0d24d524d3c093fe60a0d-6a068697363048e1be7840a49daf4820 2024-12-04T14:18:03,283 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6ac93f48bc8d46aa9990e6d670d448a8 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6ac93f48bc8d46aa9990e6d670d448a8 2024-12-04T14:18:03,284 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/eee0cbc4a66a43e1bf64a1630a49a3ee to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/eee0cbc4a66a43e1bf64a1630a49a3ee 2024-12-04T14:18:03,285 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/b7c9395f25e845e28cb35ec4f49544e0 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/b7c9395f25e845e28cb35ec4f49544e0 2024-12-04T14:18:03,286 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/712bfda33f8d416aae8eb0051e426092 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/712bfda33f8d416aae8eb0051e426092 2024-12-04T14:18:03,287 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/dfb3f437335f4cc9a295e9f50e39258e to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/dfb3f437335f4cc9a295e9f50e39258e 2024-12-04T14:18:03,288 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1cd1c0e81ec9418f9aecf49efb75dd6b to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1cd1c0e81ec9418f9aecf49efb75dd6b 2024-12-04T14:18:03,289 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0829a400952243ee9a733aedb6ed2b5c to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0829a400952243ee9a733aedb6ed2b5c 2024-12-04T14:18:03,290 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/792c45fa7307437883034cdebebf7e50 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/792c45fa7307437883034cdebebf7e50 2024-12-04T14:18:03,291 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4132a4da43fb490ead8885f87cd63196 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4132a4da43fb490ead8885f87cd63196 2024-12-04T14:18:03,292 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7a7b5159767a419fa9d430dd73566f4f to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7a7b5159767a419fa9d430dd73566f4f 2024-12-04T14:18:03,294 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/9f414277f2ec47a082755ae9b4ac5595 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/9f414277f2ec47a082755ae9b4ac5595 2024-12-04T14:18:03,295 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1a306769e18348f7b7063cb623b0af89 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/1a306769e18348f7b7063cb623b0af89 2024-12-04T14:18:03,296 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7f8c54fcd2ab42b9883b751ba907e428 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/7f8c54fcd2ab42b9883b751ba907e428 2024-12-04T14:18:03,297 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0af625f6a2ed4f89841f6755bd858364 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/0af625f6a2ed4f89841f6755bd858364 2024-12-04T14:18:03,298 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6291182bfdaf4e8887c71fa21e9d6206 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/6291182bfdaf4e8887c71fa21e9d6206 2024-12-04T14:18:03,299 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4f07f383429a42e4a88b4baf325479cf to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/4f07f383429a42e4a88b4baf325479cf 2024-12-04T14:18:03,300 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/8607be58ed9c4caaa6c5e42eff2396c4 to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/info/8607be58ed9c4caaa6c5e42eff2396c4 2024-12-04T14:18:03,304 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/fda0efb5cff38b91cf5f681856c2db39/recovered.edits/330.seqid, newMaxSeqId=330, maxSeqId=121 2024-12-04T14:18:03,305 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:18:03,305 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for fda0efb5cff38b91cf5f681856c2db39: 2024-12-04T14:18:03,305 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733321800457.fda0efb5cff38b91cf5f681856c2db39. 2024-12-04T14:18:03,305 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e69ba94e2d59fa312508f78c59f9c076, disabling compactions & flushes 2024-12-04T14:18:03,305 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:18:03,305 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:18:03,305 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. after waiting 0 ms 2024-12-04T14:18:03,305 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:18:03,305 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d->hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/939b7947bcc0d24d524d3c093fe60a0d/info/de347c398e5345d7a7d03eaad2ec0198-bottom] to archive 2024-12-04T14:18:03,306 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:18:03,308 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d to hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/archive/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/info/de347c398e5345d7a7d03eaad2ec0198.939b7947bcc0d24d524d3c093fe60a0d 2024-12-04T14:18:03,312 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/data/default/TestLogRolling-testLogRolling/e69ba94e2d59fa312508f78c59f9c076/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=121 2024-12-04T14:18:03,313 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:18:03,313 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e69ba94e2d59fa312508f78c59f9c076: 2024-12-04T14:18:03,313 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733321800457.e69ba94e2d59fa312508f78c59f9c076. 2024-12-04T14:18:03,470 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,37995,1733321774863; all regions closed. 2024-12-04T14:18:03,471 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863 2024-12-04T14:18:03,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741834_1010 (size=9351) 2024-12-04T14:18:03,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741834_1010 (size=9351) 2024-12-04T14:18:03,475 DEBUG [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/oldWALs 2024-12-04T14:18:03,475 INFO [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C37995%2C1733321774863.meta:.meta(num 1733321775618) 2024-12-04T14:18:03,475 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/WALs/ec15ac6edd88,37995,1733321774863 2024-12-04T14:18:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741877_1053 (size=1071) 2024-12-04T14:18:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741877_1053 (size=1071) 2024-12-04T14:18:03,479 DEBUG [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/oldWALs 2024-12-04T14:18:03,479 INFO [RS:0;ec15ac6edd88:37995 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C37995%2C1733321774863:(num 1733321883160) 2024-12-04T14:18:03,479 DEBUG [RS:0;ec15ac6edd88:37995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:03,479 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:18:03,479 INFO [RS:0;ec15ac6edd88:37995 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-04T14:18:03,479 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:18:03,479 INFO [RS:0;ec15ac6edd88:37995 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:37995 2024-12-04T14:18:03,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:18:03,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,37995,1733321774863 2024-12-04T14:18:03,483 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,37995,1733321774863] 2024-12-04T14:18:03,483 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,37995,1733321774863; numProcessing=1 2024-12-04T14:18:03,484 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,37995,1733321774863 already deleted, retry=false 2024-12-04T14:18:03,484 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,37995,1733321774863 expired; onlineServers=0 2024-12-04T14:18:03,484 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,40263,1733321774812' ***** 2024-12-04T14:18:03,484 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:18:03,484 DEBUG [M:0;ec15ac6edd88:40263 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9c9888, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:18:03,484 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,40263,1733321774812 2024-12-04T14:18:03,484 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,40263,1733321774812; all regions closed. 2024-12-04T14:18:03,484 DEBUG [M:0;ec15ac6edd88:40263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:03,484 DEBUG [M:0;ec15ac6edd88:40263 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:18:03,484 DEBUG [M:0;ec15ac6edd88:40263 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:18:03,484 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:18:03,484 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321775016 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321775016,5,FailOnTimeoutGroup] 2024-12-04T14:18:03,484 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321775019 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321775019,5,FailOnTimeoutGroup] 2024-12-04T14:18:03,484 INFO [M:0;ec15ac6edd88:40263 {}] hbase.ChoreService(370): Chore service for: master/ec15ac6edd88:0 had [] on shutdown 2024-12-04T14:18:03,485 DEBUG [M:0;ec15ac6edd88:40263 {}] master.HMaster(1733): Stopping service threads 2024-12-04T14:18:03,485 INFO [M:0;ec15ac6edd88:40263 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:18:03,485 INFO [M:0;ec15ac6edd88:40263 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:18:03,485 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:18:03,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:18:03,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:03,486 DEBUG [M:0;ec15ac6edd88:40263 {}] zookeeper.ZKUtil(347): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:18:03,486 WARN [M:0;ec15ac6edd88:40263 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:18:03,486 INFO [M:0;ec15ac6edd88:40263 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-04T14:18:03,486 INFO [M:0;ec15ac6edd88:40263 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:18:03,486 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:18:03,486 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:18:03,486 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:03,486 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:03,486 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:18:03,486 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:03,486 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=66.44 KB heapSize=81.69 KB 2024-12-04T14:18:03,502 DEBUG [M:0;ec15ac6edd88:40263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6414bab1c37c4d06804839aa116b7133 is 82, key is hbase:meta,,1/info:regioninfo/1733321775637/Put/seqid=0 2024-12-04T14:18:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741878_1054 (size=5672) 2024-12-04T14:18:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741878_1054 (size=5672) 2024-12-04T14:18:03,508 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6414bab1c37c4d06804839aa116b7133 2024-12-04T14:18:03,526 DEBUG [M:0;ec15ac6edd88:40263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46a377802fd849dc8f0eac2ab6eeba58 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733321776571/Put/seqid=0 2024-12-04T14:18:03,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741879_1055 (size=7285) 2024-12-04T14:18:03,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741879_1055 (size=7285) 2024-12-04T14:18:03,531 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65.83 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46a377802fd849dc8f0eac2ab6eeba58 2024-12-04T14:18:03,536 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 46a377802fd849dc8f0eac2ab6eeba58 2024-12-04T14:18:03,550 DEBUG [M:0;ec15ac6edd88:40263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38d489e70eaf40f49b091d2d411aa2c7 is 69, key is ec15ac6edd88,37995,1733321774863/rs:state/1733321775111/Put/seqid=0 2024-12-04T14:18:03,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741880_1056 (size=5156) 2024-12-04T14:18:03,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741880_1056 (size=5156) 2024-12-04T14:18:03,555 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38d489e70eaf40f49b091d2d411aa2c7 2024-12-04T14:18:03,581 DEBUG [M:0;ec15ac6edd88:40263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6e7ea28684884ddaaaf60dea191d875d is 52, key is load_balancer_on/state:d/1733321776203/Put/seqid=0 2024-12-04T14:18:03,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:03,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x1005d630fb00001, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:03,583 INFO [RS:0;ec15ac6edd88:37995 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,37995,1733321774863; zookeeper connection closed. 2024-12-04T14:18:03,583 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@257df853 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@257df853 2024-12-04T14:18:03,583 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T14:18:03,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741881_1057 (size=5056) 2024-12-04T14:18:03,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741881_1057 (size=5056) 2024-12-04T14:18:03,586 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6e7ea28684884ddaaaf60dea191d875d 2024-12-04T14:18:03,591 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6414bab1c37c4d06804839aa116b7133 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6414bab1c37c4d06804839aa116b7133 2024-12-04T14:18:03,595 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6414bab1c37c4d06804839aa116b7133, entries=8, sequenceid=164, filesize=5.5 K 2024-12-04T14:18:03,596 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46a377802fd849dc8f0eac2ab6eeba58 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/46a377802fd849dc8f0eac2ab6eeba58 2024-12-04T14:18:03,600 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 46a377802fd849dc8f0eac2ab6eeba58 2024-12-04T14:18:03,600 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/46a377802fd849dc8f0eac2ab6eeba58, entries=18, sequenceid=164, filesize=7.1 K 2024-12-04T14:18:03,601 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38d489e70eaf40f49b091d2d411aa2c7 as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/38d489e70eaf40f49b091d2d411aa2c7 2024-12-04T14:18:03,605 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/38d489e70eaf40f49b091d2d411aa2c7, entries=1, sequenceid=164, filesize=5.0 K 2024-12-04T14:18:03,606 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6e7ea28684884ddaaaf60dea191d875d as hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6e7ea28684884ddaaaf60dea191d875d 2024-12-04T14:18:03,610 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39545/user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6e7ea28684884ddaaaf60dea191d875d, entries=1, sequenceid=164, filesize=4.9 K 2024-12-04T14:18:03,611 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(3040): Finished flush of dataSize ~66.44 KB/68031, heapSize ~81.63 KB/83584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 125ms, sequenceid=164, compaction requested=false 2024-12-04T14:18:03,613 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:03,613 DEBUG [M:0;ec15ac6edd88:40263 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:18:03,613 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/837d5c99-4798-d4d7-3e4d-fa598c75364a/MasterData/WALs/ec15ac6edd88,40263,1733321774812 2024-12-04T14:18:03,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41847 is added to blk_1073741830_1006 (size=79260) 2024-12-04T14:18:03,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36493 is added to blk_1073741830_1006 (size=79260) 2024-12-04T14:18:03,616 INFO [M:0;ec15ac6edd88:40263 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-04T14:18:03,616 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:18:03,616 INFO [M:0;ec15ac6edd88:40263 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40263 2024-12-04T14:18:03,619 DEBUG [M:0;ec15ac6edd88:40263 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ec15ac6edd88,40263,1733321774812 already deleted, retry=false 2024-12-04T14:18:03,663 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:03,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:03,720 INFO [M:0;ec15ac6edd88:40263 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,40263,1733321774812; zookeeper connection closed. 2024-12-04T14:18:03,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40263-0x1005d630fb00000, quorum=127.0.0.1:54338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:03,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@434b09e4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:18:03,723 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17b5bb29{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:18:03,723 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:18:03,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4be9feb9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:18:03,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33dd8821{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir/,STOPPED} 2024-12-04T14:18:03,725 WARN [BP-832301905-172.17.0.2-1733321774141 heartbeating to localhost/127.0.0.1:39545 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:18:03,725 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:18:03,725 WARN [BP-832301905-172.17.0.2-1733321774141 heartbeating to localhost/127.0.0.1:39545 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-832301905-172.17.0.2-1733321774141 (Datanode Uuid 3c9ca7ad-63cf-4ea5-a044-6fda006a579f) service to localhost/127.0.0.1:39545 2024-12-04T14:18:03,725 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:18:03,725 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data3/current/BP-832301905-172.17.0.2-1733321774141 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:03,726 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data4/current/BP-832301905-172.17.0.2-1733321774141 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:03,726 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:18:03,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d11ec04{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:18:03,728 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17c78fce{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:18:03,728 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:18:03,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@bf6e2c7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:18:03,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@555a9157{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir/,STOPPED} 2024-12-04T14:18:03,730 WARN [BP-832301905-172.17.0.2-1733321774141 heartbeating to localhost/127.0.0.1:39545 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:18:03,730 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:18:03,730 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:18:03,730 WARN [BP-832301905-172.17.0.2-1733321774141 heartbeating to localhost/127.0.0.1:39545 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-832301905-172.17.0.2-1733321774141 (Datanode Uuid f67e5b87-0e40-4720-a29f-2036c21887c9) service to localhost/127.0.0.1:39545 2024-12-04T14:18:03,730 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data1/current/BP-832301905-172.17.0.2-1733321774141 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:03,731 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/cluster_f07c7895-c1bc-b525-9a9c-e0ae289eadf2/dfs/data/data2/current/BP-832301905-172.17.0.2-1733321774141 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:03,731 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:18:03,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34cb6972{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:18:03,737 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@191bd0c3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:18:03,737 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:18:03,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cad81e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:18:03,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1297c717{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir/,STOPPED} 2024-12-04T14:18:03,745 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:18:03,771 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-04T14:18:03,779 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=124 (was 108) - Thread LEAK? -, OpenFileDescriptor=487 (was 464) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=118 (was 120), ProcessCount=11 (was 11), AvailableMemoryMB=5938 (was 6023) 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=125, OpenFileDescriptor=487, MaxFileDescriptor=1048576, SystemLoadAverage=118, ProcessCount=11, AvailableMemoryMB=5938 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.log.dir so I do NOT create it in target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/65517550-9a71-4e31-49e7-e5bc12cc5eb5/hadoop.tmp.dir so I do NOT create it in target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368, deleteOnExit=true 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/test.cache.data in system properties and HBase conf 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:18:03,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-04T14:18:03,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:18:03,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:18:03,801 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:18:03,882 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:18:03,886 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:18:03,887 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:18:03,887 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:18:03,887 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:18:03,888 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:18:03,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7593038f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:18:03,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cd921e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:18:04,016 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292559c2{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/java.io.tmpdir/jetty-localhost-41793-hadoop-hdfs-3_4_1-tests_jar-_-any-3961225100935035664/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:18:04,017 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@59545e62{HTTP/1.1, (http/1.1)}{localhost:41793} 2024-12-04T14:18:04,017 INFO [Time-limited test {}] server.Server(415): Started @396487ms 2024-12-04T14:18:04,029 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T14:18:04,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:18:04,086 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:18:04,086 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:18:04,086 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:18:04,086 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T14:18:04,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5fd4cd29{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:18:04,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e5c24a7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:18:04,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70a87fbb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/java.io.tmpdir/jetty-localhost-46873-hadoop-hdfs-3_4_1-tests_jar-_-any-14418496008024612735/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:18:04,202 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@155e72ac{HTTP/1.1, (http/1.1)}{localhost:46873} 2024-12-04T14:18:04,202 INFO [Time-limited test {}] server.Server(415): Started @396672ms 2024-12-04T14:18:04,203 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:18:04,234 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:18:04,236 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:18:04,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:18:04,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:18:04,237 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:18:04,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3385e1f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:18:04,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f12d55f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:18:04,301 WARN [Thread-2233 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data1/current/BP-424584061-172.17.0.2-1733321883825/current, will proceed with Du for space computation calculation, 2024-12-04T14:18:04,301 WARN [Thread-2234 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data2/current/BP-424584061-172.17.0.2-1733321883825/current, will proceed with Du for space computation calculation, 2024-12-04T14:18:04,319 WARN [Thread-2212 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:18:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbe458ac665cc23c2 with lease ID 0x81e897bcef6692c1: Processing first storage report for DS-2637849b-290e-4a92-9eb1-b9c5e7fa09da from datanode DatanodeRegistration(127.0.0.1:41721, datanodeUuid=e135e0cf-15c0-4db3-93cc-f1c2fb83733a, infoPort=33525, infoSecurePort=0, ipcPort=38359, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825) 2024-12-04T14:18:04,322 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbe458ac665cc23c2 with lease ID 0x81e897bcef6692c1: from storage DS-2637849b-290e-4a92-9eb1-b9c5e7fa09da node DatanodeRegistration(127.0.0.1:41721, datanodeUuid=e135e0cf-15c0-4db3-93cc-f1c2fb83733a, infoPort=33525, infoSecurePort=0, ipcPort=38359, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:18:04,322 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbe458ac665cc23c2 with lease ID 0x81e897bcef6692c1: Processing first storage report for DS-f88e3ffc-2c43-49d8-92ba-6f07c13781a1 from datanode DatanodeRegistration(127.0.0.1:41721, datanodeUuid=e135e0cf-15c0-4db3-93cc-f1c2fb83733a, infoPort=33525, infoSecurePort=0, ipcPort=38359, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825) 2024-12-04T14:18:04,322 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbe458ac665cc23c2 with lease ID 0x81e897bcef6692c1: from storage DS-f88e3ffc-2c43-49d8-92ba-6f07c13781a1 node DatanodeRegistration(127.0.0.1:41721, datanodeUuid=e135e0cf-15c0-4db3-93cc-f1c2fb83733a, infoPort=33525, infoSecurePort=0, ipcPort=38359, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:18:04,360 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f8637f7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/java.io.tmpdir/jetty-localhost-37587-hadoop-hdfs-3_4_1-tests_jar-_-any-14192988717781255545/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:18:04,360 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5db53ee0{HTTP/1.1, (http/1.1)}{localhost:37587} 2024-12-04T14:18:04,360 INFO [Time-limited test {}] server.Server(415): Started @396831ms 2024-12-04T14:18:04,361 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:18:04,453 WARN [Thread-2260 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data4/current/BP-424584061-172.17.0.2-1733321883825/current, will proceed with Du for space computation calculation, 2024-12-04T14:18:04,453 WARN [Thread-2259 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data3/current/BP-424584061-172.17.0.2-1733321883825/current, will proceed with Du for space computation calculation, 2024-12-04T14:18:04,472 WARN [Thread-2248 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:18:04,476 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8e22cd120947a91 with lease ID 0x81e897bcef6692c2: Processing first storage report for DS-42a7d782-4d7e-4409-8804-2e08dbb37609 from datanode DatanodeRegistration(127.0.0.1:35817, datanodeUuid=91f5ef7b-6a0f-4352-9874-cabed8111156, infoPort=45433, infoSecurePort=0, ipcPort=44121, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825) 2024-12-04T14:18:04,476 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8e22cd120947a91 with lease ID 0x81e897bcef6692c2: from storage DS-42a7d782-4d7e-4409-8804-2e08dbb37609 node DatanodeRegistration(127.0.0.1:35817, datanodeUuid=91f5ef7b-6a0f-4352-9874-cabed8111156, infoPort=45433, infoSecurePort=0, ipcPort=44121, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:18:04,476 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8e22cd120947a91 with lease ID 0x81e897bcef6692c2: Processing first storage report for DS-f5c39f28-24a0-472f-bdb1-1fd8fb3a4d97 from datanode DatanodeRegistration(127.0.0.1:35817, datanodeUuid=91f5ef7b-6a0f-4352-9874-cabed8111156, infoPort=45433, infoSecurePort=0, ipcPort=44121, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825) 2024-12-04T14:18:04,476 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8e22cd120947a91 with lease ID 0x81e897bcef6692c2: from storage DS-f5c39f28-24a0-472f-bdb1-1fd8fb3a4d97 node DatanodeRegistration(127.0.0.1:35817, datanodeUuid=91f5ef7b-6a0f-4352-9874-cabed8111156, infoPort=45433, infoSecurePort=0, ipcPort=44121, storageInfo=lv=-57;cid=testClusterID;nsid=241388054;c=1733321883825), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T14:18:04,482 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8 2024-12-04T14:18:04,485 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/zookeeper_0, clientPort=53829, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:18:04,486 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53829 2024-12-04T14:18:04,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,488 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:18:04,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:18:04,497 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac with version=8 2024-12-04T14:18:04,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37417/user/jenkins/test-data/005bc2b5-3a98-8361-916b-2bc49ed28d78/hbase-staging 2024-12-04T14:18:04,499 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:18:04,500 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45963 2024-12-04T14:18:04,501 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,504 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:45963 connecting to ZooKeeper ensemble=127.0.0.1:53829 2024-12-04T14:18:04,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459630x0, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:18:04,517 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45963-0x1005d64bc2a0000 connected 2024-12-04T14:18:04,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:18:04,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:18:04,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:18:04,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45963 2024-12-04T14:18:04,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45963 2024-12-04T14:18:04,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45963 2024-12-04T14:18:04,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45963 2024-12-04T14:18:04,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45963 2024-12-04T14:18:04,545 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac, hbase.cluster.distributed=false 2024-12-04T14:18:04,561 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ec15ac6edd88:0 server-side Connection retries=45 2024-12-04T14:18:04,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:18:04,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:18:04,561 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:18:04,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:18:04,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:18:04,561 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:18:04,562 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:18:04,562 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:39493 2024-12-04T14:18:04,563 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:18:04,564 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:18:04,564 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,568 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:39493 connecting to ZooKeeper ensemble=127.0.0.1:53829 2024-12-04T14:18:04,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394930x0, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:18:04,571 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39493-0x1005d64bc2a0001 connected 2024-12-04T14:18:04,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:18:04,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:18:04,572 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:18:04,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39493 2024-12-04T14:18:04,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39493 2024-12-04T14:18:04,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39493 2024-12-04T14:18:04,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39493 2024-12-04T14:18:04,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39493 2024-12-04T14:18:04,574 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:18:04,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:18:04,575 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:18:04,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:18:04,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,577 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:18:04,578 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec15ac6edd88,45963,1733321884499 from backup master directory 2024-12-04T14:18:04,580 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:18:04,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:18:04,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:18:04,581 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:18:04,581 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,586 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec15ac6edd88:45963 2024-12-04T14:18:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:18:04,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:18:04,595 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/hbase.id with ID: dafc4bfd-e6c1-442c-a542-eb32cdcbe886 2024-12-04T14:18:04,604 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:18:04,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:18:04,615 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:18:04,616 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:18:04,616 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:18:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:18:04,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:18:04,624 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store 2024-12-04T14:18:04,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:18:04,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:18:04,631 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:18:04,631 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:18:04,631 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:04,631 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:04,631 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:18:04,631 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:04,631 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:04,631 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:18:04,632 WARN [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/.initializing 2024-12-04T14:18:04,632 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/WALs/ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,635 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C45963%2C1733321884499, suffix=, logDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/WALs/ec15ac6edd88,45963,1733321884499, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/oldWALs, maxLogs=10 2024-12-04T14:18:04,635 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C45963%2C1733321884499.1733321884635 2024-12-04T14:18:04,645 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/WALs/ec15ac6edd88,45963,1733321884499/ec15ac6edd88%2C45963%2C1733321884499.1733321884635 2024-12-04T14:18:04,645 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33525:33525),(127.0.0.1/127.0.0.1:45433:45433)] 2024-12-04T14:18:04,645 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:18:04,645 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:18:04,646 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,646 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:18:04,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:04,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:18:04,650 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:18:04,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:18:04,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:18:04,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:18:04,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:18:04,654 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,654 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,656 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:18:04,657 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:18:04,659 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:18:04,660 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732442, jitterRate=-0.06865222752094269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:18:04,661 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:18:04,662 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:18:04,663 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:04,665 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@535d6feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:18:04,666 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-04T14:18:04,666 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:18:04,666 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:18:04,666 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:18:04,666 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T14:18:04,667 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-04T14:18:04,667 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:18:04,669 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:18:04,670 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:18:04,671 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:18:04,672 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:18:04,672 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:18:04,675 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:18:04,676 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:18:04,676 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:18:04,681 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:18:04,682 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:18:04,683 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:18:04,685 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:18:04,686 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:18:04,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:18:04,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:18:04,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,688 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ec15ac6edd88,45963,1733321884499, sessionid=0x1005d64bc2a0000, setting cluster-up flag (Was=false) 2024-12-04T14:18:04,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,697 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:18:04,698 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:04,706 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:18:04,707 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:04,709 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-04T14:18:04,709 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-04T14:18:04,709 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:18:04,709 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec15ac6edd88,45963,1733321884499 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec15ac6edd88:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:18:04,710 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733321914711 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:18:04,711 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:18:04,711 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:18:04,711 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:18:04,712 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321884712,5,FailOnTimeoutGroup] 2024-12-04T14:18:04,712 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321884712,5,FailOnTimeoutGroup] 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:18:04,712 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,712 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,712 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:18:04,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:18:04,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741831_1007 (size=1039) 2024-12-04T14:18:04,720 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-04T14:18:04,720 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac 2024-12-04T14:18:04,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:18:04,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741832_1008 (size=32) 2024-12-04T14:18:04,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:18:04,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:18:04,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:18:04,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:04,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:18:04,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:18:04,731 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:04,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:18:04,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:18:04,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:04,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:04,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740 2024-12-04T14:18:04,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740 2024-12-04T14:18:04,735 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:18:04,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:18:04,738 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:18:04,738 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=783736, jitterRate=-0.003428354859352112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:18:04,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:18:04,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:18:04,738 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:18:04,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:18:04,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:18:04,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:18:04,739 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:18:04,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:18:04,740 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:18:04,740 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-04T14:18:04,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:18:04,741 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:18:04,741 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:18:04,785 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec15ac6edd88:39493 2024-12-04T14:18:04,786 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1008): ClusterId : dafc4bfd-e6c1-442c-a542-eb32cdcbe886 2024-12-04T14:18:04,786 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:18:04,790 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:18:04,790 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:18:04,795 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:18:04,796 DEBUG [RS:0;ec15ac6edd88:39493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b63ad27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:18:04,796 DEBUG [RS:0;ec15ac6edd88:39493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f0db50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:18:04,796 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-04T14:18:04,796 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-04T14:18:04,796 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-04T14:18:04,797 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(3073): reportForDuty to master=ec15ac6edd88,45963,1733321884499 with isa=ec15ac6edd88/172.17.0.2:39493, startcode=1733321884560 2024-12-04T14:18:04,797 DEBUG [RS:0;ec15ac6edd88:39493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:18:04,799 INFO [RS-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37119, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:18:04,799 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45963 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:04,799 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45963 {}] master.ServerManager(486): Registering regionserver=ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:04,800 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac 2024-12-04T14:18:04,800 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37857 2024-12-04T14:18:04,800 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-04T14:18:04,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:18:04,802 DEBUG [RS:0;ec15ac6edd88:39493 {}] zookeeper.ZKUtil(111): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:04,802 WARN [RS:0;ec15ac6edd88:39493 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:18:04,802 INFO [RS:0;ec15ac6edd88:39493 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:18:04,802 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:04,802 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec15ac6edd88,39493,1733321884560] 2024-12-04T14:18:04,805 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-04T14:18:04,805 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:18:04,806 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:18:04,807 INFO [RS:0;ec15ac6edd88:39493 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:18:04,807 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,807 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-04T14:18:04,808 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec15ac6edd88:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec15ac6edd88:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:18:04,808 DEBUG [RS:0;ec15ac6edd88:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec15ac6edd88:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:18:04,809 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,809 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,809 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,809 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,809 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,39493,1733321884560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:18:04,823 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:18:04,823 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,39493,1733321884560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:04,837 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.Replication(204): ec15ac6edd88,39493,1733321884560 started 2024-12-04T14:18:04,838 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1767): Serving as ec15ac6edd88,39493,1733321884560, RpcServer on ec15ac6edd88/172.17.0.2:39493, sessionid=0x1005d64bc2a0001 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,39493,1733321884560' 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec15ac6edd88,39493,1733321884560' 2024-12-04T14:18:04,838 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:18:04,839 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:18:04,839 DEBUG [RS:0;ec15ac6edd88:39493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:18:04,839 INFO [RS:0;ec15ac6edd88:39493 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:18:04,839 INFO [RS:0;ec15ac6edd88:39493 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:18:04,891 WARN [ec15ac6edd88:45963 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-04T14:18:04,941 INFO [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C39493%2C1733321884560, suffix=, logDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs, maxLogs=32 2024-12-04T14:18:04,942 INFO [RS:0;ec15ac6edd88:39493 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C39493%2C1733321884560.1733321884941 2024-12-04T14:18:04,947 INFO [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560/ec15ac6edd88%2C39493%2C1733321884560.1733321884941 2024-12-04T14:18:04,947 DEBUG [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33525:33525),(127.0.0.1/127.0.0.1:45433:45433)] 2024-12-04T14:18:05,125 INFO [regionserver/ec15ac6edd88:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:18:05,142 DEBUG [ec15ac6edd88:45963 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:18:05,142 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,143 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,39493,1733321884560, state=OPENING 2024-12-04T14:18:05,144 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:18:05,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:05,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:05,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ec15ac6edd88,39493,1733321884560}] 2024-12-04T14:18:05,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:18:05,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:18:05,298 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,298 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:18:05,300 INFO [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52960, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:18:05,304 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-04T14:18:05,304 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:18:05,305 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec15ac6edd88%2C39493%2C1733321884560.meta, suffix=.meta, logDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs, maxLogs=32 2024-12-04T14:18:05,307 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ec15ac6edd88%2C39493%2C1733321884560.meta.1733321885307.meta 2024-12-04T14:18:05,314 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560/ec15ac6edd88%2C39493%2C1733321884560.meta.1733321885307.meta 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33525:33525),(127.0.0.1/127.0.0.1:45433:45433)] 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:18:05,315 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-04T14:18:05,315 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-04T14:18:05,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:18:05,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:18:05,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:05,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:05,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:18:05,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:18:05,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:05,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:05,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:18:05,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:18:05,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:05,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:18:05,320 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740 2024-12-04T14:18:05,321 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740 2024-12-04T14:18:05,323 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:18:05,324 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-04T14:18:05,325 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848669, jitterRate=0.07913851737976074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:18:05,325 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-04T14:18:05,326 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733321885298 2024-12-04T14:18:05,327 DEBUG [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:18:05,327 INFO [RS_OPEN_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-04T14:18:05,328 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,329 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec15ac6edd88,39493,1733321884560, state=OPEN 2024-12-04T14:18:05,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:18:05,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:18:05,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:18:05,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:18:05,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:18:05,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ec15ac6edd88,39493,1733321884560 in 187 msec 2024-12-04T14:18:05,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:18:05,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 595 msec 2024-12-04T14:18:05,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 629 msec 2024-12-04T14:18:05,338 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733321885338, completionTime=-1 2024-12-04T14:18:05,338 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:18:05,338 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-04T14:18:05,339 DEBUG [hconnection-0x178fc063-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:18:05,340 INFO [RS-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52964, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:18:05,341 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-04T14:18:05,341 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733321945341 2024-12-04T14:18:05,341 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733322005341 2024-12-04T14:18:05,341 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45963,1733321884499-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45963,1733321884499-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45963,1733321884499-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec15ac6edd88:45963, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-04T14:18:05,346 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:18:05,347 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-04T14:18:05,347 DEBUG [master/ec15ac6edd88:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-04T14:18:05,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:18:05,348 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:05,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:18:05,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:18:05,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741835_1011 (size=358) 2024-12-04T14:18:05,358 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6fa4042f64bac9ec83a9d0c486b001bf, NAME => 'hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac 2024-12-04T14:18:05,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:18:05,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741836_1012 (size=42) 2024-12-04T14:18:05,364 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:18:05,364 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 6fa4042f64bac9ec83a9d0c486b001bf, disabling compactions & flushes 2024-12-04T14:18:05,364 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,364 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. after waiting 0 ms 2024-12-04T14:18:05,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,365 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6fa4042f64bac9ec83a9d0c486b001bf: 2024-12-04T14:18:05,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:18:05,366 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733321885366"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733321885366"}]},"ts":"1733321885366"} 2024-12-04T14:18:05,367 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-04T14:18:05,368 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:18:05,368 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321885368"}]},"ts":"1733321885368"} 2024-12-04T14:18:05,369 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-04T14:18:05,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=6fa4042f64bac9ec83a9d0c486b001bf, ASSIGN}] 2024-12-04T14:18:05,373 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=6fa4042f64bac9ec83a9d0c486b001bf, ASSIGN 2024-12-04T14:18:05,374 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=6fa4042f64bac9ec83a9d0c486b001bf, ASSIGN; state=OFFLINE, location=ec15ac6edd88,39493,1733321884560; forceNewPlan=false, retain=false 2024-12-04T14:18:05,524 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=6fa4042f64bac9ec83a9d0c486b001bf, regionState=OPENING, regionLocation=ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 6fa4042f64bac9ec83a9d0c486b001bf, server=ec15ac6edd88,39493,1733321884560}] 2024-12-04T14:18:05,664 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37629/user/jenkins/test-data/62f9cdac-8222-1912-75f7-c3af41f55cb8/WALs/ec15ac6edd88,40113,1733321604895/ec15ac6edd88%2C40113%2C1733321604895.meta.1733321606144.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T14:18:05,678 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,682 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 6fa4042f64bac9ec83a9d0c486b001bf, NAME => 'hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:18:05,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:18:05,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,683 INFO [StoreOpener-6fa4042f64bac9ec83a9d0c486b001bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,684 INFO [StoreOpener-6fa4042f64bac9ec83a9d0c486b001bf-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fa4042f64bac9ec83a9d0c486b001bf columnFamilyName info 2024-12-04T14:18:05,685 DEBUG [StoreOpener-6fa4042f64bac9ec83a9d0c486b001bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:18:05,685 INFO [StoreOpener-6fa4042f64bac9ec83a9d0c486b001bf-1 {}] regionserver.HStore(327): Store=6fa4042f64bac9ec83a9d0c486b001bf/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:18:05,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,690 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:18:05,690 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 6fa4042f64bac9ec83a9d0c486b001bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=814720, jitterRate=0.03597019612789154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T14:18:05,690 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 6fa4042f64bac9ec83a9d0c486b001bf: 2024-12-04T14:18:05,691 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf., pid=6, masterSystemTime=1733321885678 2024-12-04T14:18:05,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,692 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,693 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=6fa4042f64bac9ec83a9d0c486b001bf, regionState=OPEN, openSeqNum=2, regionLocation=ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:18:05,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 6fa4042f64bac9ec83a9d0c486b001bf, server=ec15ac6edd88,39493,1733321884560 in 168 msec 2024-12-04T14:18:05,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:18:05,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=6fa4042f64bac9ec83a9d0c486b001bf, ASSIGN in 324 msec 2024-12-04T14:18:05,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:18:05,698 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733321885698"}]},"ts":"1733321885698"} 2024-12-04T14:18:05,699 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-04T14:18:05,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:18:05,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 356 msec 2024-12-04T14:18:05,748 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-04T14:18:05,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:18:05,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:05,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:05,754 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-04T14:18:05,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:18:05,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 8 msec 2024-12-04T14:18:05,765 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-04T14:18:05,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-04T14:18:05,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 7 msec 2024-12-04T14:18:05,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-04T14:18:05,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.200sec 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:18:05,781 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45963,1733321884499-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:18:05,782 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45963,1733321884499-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:18:05,783 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:18:05,783 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:18:05,783 INFO [master/ec15ac6edd88:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec15ac6edd88,45963,1733321884499-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:18:05,876 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e09df6d to 127.0.0.1:53829 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15a8a0af 2024-12-04T14:18:05,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c99297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:18:05,880 DEBUG [hconnection-0x352fef5d-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:18:05,882 INFO [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52980, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:18:05,883 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:05,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:18:05,885 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-04T14:18:05,885 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T14:18:05,887 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs, maxLogs=32 2024-12-04T14:18:05,888 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733321885888 2024-12-04T14:18:05,892 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1/test.com%2C8080%2C1.1733321885888 2024-12-04T14:18:05,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33525:33525),(127.0.0.1/127.0.0.1:45433:45433)] 2024-12-04T14:18:05,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733321885892 2024-12-04T14:18:05,898 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1/test.com%2C8080%2C1.1733321885888 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1/test.com%2C8080%2C1.1733321885892 2024-12-04T14:18:05,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33525:33525),(127.0.0.1/127.0.0.1:45433:45433)] 2024-12-04T14:18:05,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1/test.com%2C8080%2C1.1733321885888 is not closed yet, will try archiving it next time 2024-12-04T14:18:05,898 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1 2024-12-04T14:18:05,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741837_1013 (size=93) 2024-12-04T14:18:05,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741837_1013 (size=93) 2024-12-04T14:18:05,900 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/test.com,8080,1/test.com%2C8080%2C1.1733321885888 to hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs/test.com%2C8080%2C1.1733321885888 2024-12-04T14:18:05,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741838_1014 (size=93) 2024-12-04T14:18:05,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741838_1014 (size=93) 2024-12-04T14:18:05,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs 2024-12-04T14:18:05,904 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733321885892) 2024-12-04T14:18:05,904 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-04T14:18:05,904 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e09df6d to 127.0.0.1:53829 2024-12-04T14:18:05,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:05,904 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:18:05,904 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1308784511, stopped=false 2024-12-04T14:18:05,904 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:05,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:18:05,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:05,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:18:05,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:05,906 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-04T14:18:05,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:18:05,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:18:05,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:05,907 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,39493,1733321884560' ***** 2024-12-04T14:18:05,907 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(3579): Received CLOSE for 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,908 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:05,908 DEBUG [RS:0;ec15ac6edd88:39493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:18:05,908 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6fa4042f64bac9ec83a9d0c486b001bf, disabling compactions & flushes 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-04T14:18:05,908 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,908 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,908 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. after waiting 0 ms 2024-12-04T14:18:05,908 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,908 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6fa4042f64bac9ec83a9d0c486b001bf 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-04T14:18:05,908 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-04T14:18:05,909 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 6fa4042f64bac9ec83a9d0c486b001bf=hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf.} 2024-12-04T14:18:05,909 DEBUG [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 6fa4042f64bac9ec83a9d0c486b001bf 2024-12-04T14:18:05,909 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:18:05,909 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-04T14:18:05,909 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-04T14:18:05,909 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:18:05,909 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:18:05,909 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-12-04T14:18:05,924 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/.tmp/info/d51d6e9628e64b74a7128dc4f0826e73 is 45, key is default/info:d/1733321885757/Put/seqid=0 2024-12-04T14:18:05,924 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/.tmp/info/29bfc2526da547289e98bd6de7334fe3 is 143, key is hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf./info:regioninfo/1733321885693/Put/seqid=0 2024-12-04T14:18:05,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741839_1015 (size=5037) 2024-12-04T14:18:05,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741839_1015 (size=5037) 2024-12-04T14:18:05,930 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/.tmp/info/d51d6e9628e64b74a7128dc4f0826e73 2024-12-04T14:18:05,935 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/.tmp/info/d51d6e9628e64b74a7128dc4f0826e73 as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/info/d51d6e9628e64b74a7128dc4f0826e73 2024-12-04T14:18:05,939 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/info/d51d6e9628e64b74a7128dc4f0826e73, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T14:18:05,940 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 6fa4042f64bac9ec83a9d0c486b001bf in 31ms, sequenceid=6, compaction requested=false 2024-12-04T14:18:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741840_1016 (size=6595) 2024-12-04T14:18:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741840_1016 (size=6595) 2024-12-04T14:18:05,942 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/.tmp/info/29bfc2526da547289e98bd6de7334fe3 2024-12-04T14:18:05,951 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/namespace/6fa4042f64bac9ec83a9d0c486b001bf/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T14:18:05,952 INFO [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,952 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6fa4042f64bac9ec83a9d0c486b001bf: 2024-12-04T14:18:05,952 DEBUG [RS_CLOSE_REGION-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733321885346.6fa4042f64bac9ec83a9d0c486b001bf. 2024-12-04T14:18:05,967 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/.tmp/table/08c276b419b64b19b847b20dd11f83c5 is 51, key is hbase:namespace/table:state/1733321885698/Put/seqid=0 2024-12-04T14:18:05,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741841_1017 (size=5242) 2024-12-04T14:18:05,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741841_1017 (size=5242) 2024-12-04T14:18:05,972 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/.tmp/table/08c276b419b64b19b847b20dd11f83c5 2024-12-04T14:18:05,977 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/.tmp/info/29bfc2526da547289e98bd6de7334fe3 as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/info/29bfc2526da547289e98bd6de7334fe3 2024-12-04T14:18:05,983 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/info/29bfc2526da547289e98bd6de7334fe3, entries=10, sequenceid=9, filesize=6.4 K 2024-12-04T14:18:05,984 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/.tmp/table/08c276b419b64b19b847b20dd11f83c5 as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/table/08c276b419b64b19b847b20dd11f83c5 2024-12-04T14:18:05,988 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/table/08c276b419b64b19b847b20dd11f83c5, entries=2, sequenceid=9, filesize=5.1 K 2024-12-04T14:18:05,989 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=9, compaction requested=false 2024-12-04T14:18:05,993 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-04T14:18:05,994 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:18:05,994 INFO [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-04T14:18:05,994 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-04T14:18:05,994 DEBUG [RS_CLOSE_META-regionserver/ec15ac6edd88:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T14:18:06,109 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,39493,1733321884560; all regions closed. 2024-12-04T14:18:06,109 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:06,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741834_1010 (size=2484) 2024-12-04T14:18:06,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741834_1010 (size=2484) 2024-12-04T14:18:06,113 DEBUG [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs 2024-12-04T14:18:06,113 INFO [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C39493%2C1733321884560.meta:.meta(num 1733321885307) 2024-12-04T14:18:06,113 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/WALs/ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:06,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741833_1009 (size=1414) 2024-12-04T14:18:06,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741833_1009 (size=1414) 2024-12-04T14:18:06,117 DEBUG [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/oldWALs 2024-12-04T14:18:06,117 INFO [RS:0;ec15ac6edd88:39493 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog ec15ac6edd88%2C39493%2C1733321884560:(num 1733321884941) 2024-12-04T14:18:06,117 DEBUG [RS:0;ec15ac6edd88:39493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:06,117 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:18:06,117 INFO [RS:0;ec15ac6edd88:39493 {}] hbase.ChoreService(370): Chore service for: regionserver/ec15ac6edd88:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T14:18:06,118 INFO [regionserver/ec15ac6edd88:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:18:06,118 INFO [RS:0;ec15ac6edd88:39493 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:39493 2024-12-04T14:18:06,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:18:06,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec15ac6edd88,39493,1733321884560 2024-12-04T14:18:06,121 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec15ac6edd88,39493,1733321884560] 2024-12-04T14:18:06,121 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ec15ac6edd88,39493,1733321884560; numProcessing=1 2024-12-04T14:18:06,122 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ec15ac6edd88,39493,1733321884560 already deleted, retry=false 2024-12-04T14:18:06,122 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ec15ac6edd88,39493,1733321884560 expired; onlineServers=0 2024-12-04T14:18:06,122 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ec15ac6edd88,45963,1733321884499' ***** 2024-12-04T14:18:06,122 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:18:06,122 DEBUG [M:0;ec15ac6edd88:45963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33eaf90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec15ac6edd88/172.17.0.2:0 2024-12-04T14:18:06,122 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionServer(1224): stopping server ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:06,122 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionServer(1250): stopping server ec15ac6edd88,45963,1733321884499; all regions closed. 2024-12-04T14:18:06,122 DEBUG [M:0;ec15ac6edd88:45963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:18:06,122 DEBUG [M:0;ec15ac6edd88:45963 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:18:06,122 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:18:06,122 DEBUG [M:0;ec15ac6edd88:45963 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:18:06,122 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321884712 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.large.0-1733321884712,5,FailOnTimeoutGroup] 2024-12-04T14:18:06,122 DEBUG [master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321884712 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec15ac6edd88:0:becomeActiveMaster-HFileCleaner.small.0-1733321884712,5,FailOnTimeoutGroup] 2024-12-04T14:18:06,123 INFO [M:0;ec15ac6edd88:45963 {}] hbase.ChoreService(370): Chore service for: master/ec15ac6edd88:0 had [] on shutdown 2024-12-04T14:18:06,123 DEBUG [M:0;ec15ac6edd88:45963 {}] master.HMaster(1733): Stopping service threads 2024-12-04T14:18:06,123 INFO [M:0;ec15ac6edd88:45963 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:18:06,123 INFO [M:0;ec15ac6edd88:45963 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:18:06,123 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:18:06,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:18:06,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:18:06,124 DEBUG [M:0;ec15ac6edd88:45963 {}] zookeeper.ZKUtil(347): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:18:06,124 WARN [M:0;ec15ac6edd88:45963 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:18:06,124 INFO [M:0;ec15ac6edd88:45963 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-04T14:18:06,124 INFO [M:0;ec15ac6edd88:45963 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:18:06,124 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:18:06,124 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:18:06,124 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:06,124 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:06,124 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:18:06,124 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:06,124 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.32 KB heapSize=32.31 KB 2024-12-04T14:18:06,140 DEBUG [M:0;ec15ac6edd88:45963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc62bd1a14e24a098f93d0ad383c6d08 is 82, key is hbase:meta,,1/info:regioninfo/1733321885328/Put/seqid=0 2024-12-04T14:18:06,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741842_1018 (size=5672) 2024-12-04T14:18:06,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741842_1018 (size=5672) 2024-12-04T14:18:06,145 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc62bd1a14e24a098f93d0ad383c6d08 2024-12-04T14:18:06,164 DEBUG [M:0;ec15ac6edd88:45963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59e226ccd33e430f9a7bffb739d99abe is 696, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733321885702/Put/seqid=0 2024-12-04T14:18:06,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741843_1019 (size=6626) 2024-12-04T14:18:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741843_1019 (size=6626) 2024-12-04T14:18:06,169 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.72 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59e226ccd33e430f9a7bffb739d99abe 2024-12-04T14:18:06,187 DEBUG [M:0;ec15ac6edd88:45963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/17c6f4d2a4834337a55d1a117a3580bc is 69, key is ec15ac6edd88,39493,1733321884560/rs:state/1733321884799/Put/seqid=0 2024-12-04T14:18:06,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741844_1020 (size=5156) 2024-12-04T14:18:06,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741844_1020 (size=5156) 2024-12-04T14:18:06,192 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/17c6f4d2a4834337a55d1a117a3580bc 2024-12-04T14:18:06,210 DEBUG [M:0;ec15ac6edd88:45963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4a13c8d5e39407ab25357a2a18d0762 is 52, key is load_balancer_on/state:d/1733321885884/Put/seqid=0 2024-12-04T14:18:06,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741845_1021 (size=5056) 2024-12-04T14:18:06,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741845_1021 (size=5056) 2024-12-04T14:18:06,216 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4a13c8d5e39407ab25357a2a18d0762 2024-12-04T14:18:06,221 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc62bd1a14e24a098f93d0ad383c6d08 as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc62bd1a14e24a098f93d0ad383c6d08 2024-12-04T14:18:06,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:06,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1005d64bc2a0001, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:06,221 INFO [RS:0;ec15ac6edd88:39493 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,39493,1733321884560; zookeeper connection closed. 2024-12-04T14:18:06,221 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@34ea21a5 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@34ea21a5 2024-12-04T14:18:06,221 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T14:18:06,225 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc62bd1a14e24a098f93d0ad383c6d08, entries=8, sequenceid=70, filesize=5.5 K 2024-12-04T14:18:06,226 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59e226ccd33e430f9a7bffb739d99abe as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/59e226ccd33e430f9a7bffb739d99abe 2024-12-04T14:18:06,231 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/59e226ccd33e430f9a7bffb739d99abe, entries=8, sequenceid=70, filesize=6.5 K 2024-12-04T14:18:06,231 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/17c6f4d2a4834337a55d1a117a3580bc as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/17c6f4d2a4834337a55d1a117a3580bc 2024-12-04T14:18:06,236 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/17c6f4d2a4834337a55d1a117a3580bc, entries=1, sequenceid=70, filesize=5.0 K 2024-12-04T14:18:06,237 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4a13c8d5e39407ab25357a2a18d0762 as hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c4a13c8d5e39407ab25357a2a18d0762 2024-12-04T14:18:06,241 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37857/user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c4a13c8d5e39407ab25357a2a18d0762, entries=1, sequenceid=70, filesize=4.9 K 2024-12-04T14:18:06,242 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.32 KB/25929, heapSize ~32.25 KB/33024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=70, compaction requested=false 2024-12-04T14:18:06,244 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:18:06,244 DEBUG [M:0;ec15ac6edd88:45963 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-04T14:18:06,244 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/82d59e1b-9bb1-3602-b7b1-bf6d82f646ac/MasterData/WALs/ec15ac6edd88,45963,1733321884499 2024-12-04T14:18:06,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35817 is added to blk_1073741830_1006 (size=31030) 2024-12-04T14:18:06,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41721 is added to blk_1073741830_1006 (size=31030) 2024-12-04T14:18:06,246 INFO [M:0;ec15ac6edd88:45963 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-04T14:18:06,246 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-04T14:18:06,247 INFO [M:0;ec15ac6edd88:45963 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45963 2024-12-04T14:18:06,249 DEBUG [M:0;ec15ac6edd88:45963 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ec15ac6edd88,45963,1733321884499 already deleted, retry=false 2024-12-04T14:18:06,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:06,350 INFO [M:0;ec15ac6edd88:45963 {}] regionserver.HRegionServer(1307): Exiting; stopping=ec15ac6edd88,45963,1733321884499; zookeeper connection closed. 2024-12-04T14:18:06,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45963-0x1005d64bc2a0000, quorum=127.0.0.1:53829, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:18:06,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f8637f7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:18:06,354 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5db53ee0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:18:06,354 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:18:06,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f12d55f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:18:06,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3385e1f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir/,STOPPED} 2024-12-04T14:18:06,355 WARN [BP-424584061-172.17.0.2-1733321883825 heartbeating to localhost/127.0.0.1:37857 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:18:06,355 WARN [BP-424584061-172.17.0.2-1733321883825 heartbeating to localhost/127.0.0.1:37857 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-424584061-172.17.0.2-1733321883825 (Datanode Uuid 91f5ef7b-6a0f-4352-9874-cabed8111156) service to localhost/127.0.0.1:37857 2024-12-04T14:18:06,355 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:18:06,355 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:18:06,356 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data3/current/BP-424584061-172.17.0.2-1733321883825 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:06,356 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data4/current/BP-424584061-172.17.0.2-1733321883825 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:06,356 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:18:06,358 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70a87fbb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:18:06,359 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@155e72ac{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:18:06,359 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:18:06,359 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e5c24a7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:18:06,359 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5fd4cd29{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir/,STOPPED} 2024-12-04T14:18:06,360 WARN [BP-424584061-172.17.0.2-1733321883825 heartbeating to localhost/127.0.0.1:37857 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:18:06,360 WARN [BP-424584061-172.17.0.2-1733321883825 heartbeating to localhost/127.0.0.1:37857 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-424584061-172.17.0.2-1733321883825 (Datanode Uuid e135e0cf-15c0-4db3-93cc-f1c2fb83733a) service to localhost/127.0.0.1:37857 2024-12-04T14:18:06,360 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:18:06,360 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:18:06,361 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data1/current/BP-424584061-172.17.0.2-1733321883825 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:06,361 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/cluster_1bd6bb04-e52e-e936-9848-58af06674368/dfs/data/data2/current/BP-424584061-172.17.0.2-1733321883825 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:18:06,361 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:18:06,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292559c2{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:18:06,369 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@59545e62{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:18:06,369 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:18:06,369 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cd921e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:18:06,369 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7593038f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9af25721-9f0f-1a4e-cad7-4f0ea3d08df8/hadoop.log.dir/,STOPPED} 2024-12-04T14:18:06,375 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:18:06,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-04T14:18:06,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=146 (was 125) - Thread LEAK? -, OpenFileDescriptor=516 (was 487) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 118) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5929 (was 5938)