2024-12-15 06:12:04,863 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc 2024-12-15 06:12:04,895 main DEBUG Took 0.023885 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-15 06:12:04,896 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-15 06:12:04,896 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-15 06:12:04,897 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-15 06:12:04,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,910 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-15 06:12:04,936 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,939 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,941 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,942 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,949 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,952 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,954 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,956 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,958 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,960 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,962 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-15 06:12:04,964 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,964 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-15 06:12:04,967 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-15 06:12:04,969 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-15 06:12:04,972 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-15 06:12:04,973 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-15 06:12:04,974 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-15 06:12:04,975 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-15 06:12:04,987 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-15 06:12:04,990 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-15 06:12:04,992 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-15 06:12:04,993 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-15 06:12:04,993 main DEBUG createAppenders(={Console}) 2024-12-15 06:12:04,997 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc initialized 2024-12-15 06:12:04,997 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc 2024-12-15 06:12:04,997 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc OK. 2024-12-15 06:12:04,998 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-15 06:12:04,998 main DEBUG OutputStream closed 2024-12-15 06:12:04,999 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-15 06:12:04,999 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-15 06:12:04,999 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53ce1329 OK 2024-12-15 06:12:05,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-15 06:12:05,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-15 06:12:05,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-15 06:12:05,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-15 06:12:05,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-15 06:12:05,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-15 06:12:05,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-15 06:12:05,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-15 06:12:05,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-15 06:12:05,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-15 06:12:05,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-15 06:12:05,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-15 06:12:05,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-15 06:12:05,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-15 06:12:05,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-15 06:12:05,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-15 06:12:05,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-15 06:12:05,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-15 06:12:05,189 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-15 06:12:05,190 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-12-15 06:12:05,191 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-15 06:12:05,191 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-12-15T06:12:05,643 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39 2024-12-15 06:12:05,653 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-15 06:12:05,654 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-15T06:12:05,669 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-12-15T06:12:05,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=286, MaxFileDescriptor=1048576, SystemLoadAverage=656, ProcessCount=11, AvailableMemoryMB=3285 2024-12-15T06:12:05,728 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-15T06:12:05,735 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a, deleteOnExit=true 2024-12-15T06:12:05,736 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-15T06:12:05,740 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/test.cache.data in system properties and HBase conf 2024-12-15T06:12:05,742 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.tmp.dir in system properties and HBase conf 2024-12-15T06:12:05,744 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir in system properties and HBase conf 2024-12-15T06:12:05,745 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-15T06:12:05,746 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-15T06:12:05,746 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-15T06:12:05,983 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-15T06:12:06,220 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-15T06:12:06,227 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:12:06,228 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:12:06,228 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-15T06:12:06,229 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:12:06,230 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-15T06:12:06,230 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-15T06:12:06,231 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:12:06,232 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:12:06,232 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-15T06:12:06,233 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/nfs.dump.dir in system properties and HBase conf 2024-12-15T06:12:06,233 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/java.io.tmpdir in system properties and HBase conf 2024-12-15T06:12:06,234 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:12:06,234 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-15T06:12:06,235 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-15T06:12:07,401 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:12:08,627 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-15T06:12:08,714 INFO [Time-limited test {}] log.Log(170): Logging initialized @5842ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-15T06:12:08,810 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:12:08,893 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:12:08,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:12:08,917 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:12:08,918 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:12:08,932 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:12:08,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1342c19c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:12:08,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65cdbba1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:12:09,173 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@dfacfed{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/java.io.tmpdir/jetty-localhost-36483-hadoop-hdfs-3_4_1-tests_jar-_-any-11714336765770862728/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:12:09,183 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6483b5b2{HTTP/1.1, (http/1.1)}{localhost:36483} 2024-12-15T06:12:09,184 INFO [Time-limited test {}] server.Server(415): Started @6313ms 2024-12-15T06:12:09,220 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:12:10,837 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:12:10,849 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:12:10,850 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:12:10,851 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:12:10,851 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:12:10,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16f0f758{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:12:10,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53ad73b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:12:11,129 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6170d8a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/java.io.tmpdir/jetty-localhost-45075-hadoop-hdfs-3_4_1-tests_jar-_-any-10418553872753054900/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:12:11,130 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b349526{HTTP/1.1, (http/1.1)}{localhost:45075} 2024-12-15T06:12:11,131 INFO [Time-limited test {}] server.Server(415): Started @8259ms 2024-12-15T06:12:11,244 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:12:11,549 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:12:11,554 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:12:11,555 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:12:11,555 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:12:11,555 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:12:11,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a80aaee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:12:11,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5782a133{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:12:11,772 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42348d05{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/java.io.tmpdir/jetty-localhost-39161-hadoop-hdfs-3_4_1-tests_jar-_-any-17050580332487434196/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:12:11,773 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53406647{HTTP/1.1, (http/1.1)}{localhost:39161} 2024-12-15T06:12:11,774 INFO [Time-limited test {}] server.Server(415): Started @8902ms 2024-12-15T06:12:11,776 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:12:12,985 WARN [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data2/current/BP-125853089-172.17.0.2-1734243127551/current, will proceed with Du for space computation calculation, 2024-12-15T06:12:12,985 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data3/current/BP-125853089-172.17.0.2-1734243127551/current, will proceed with Du for space computation calculation, 2024-12-15T06:12:12,985 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data4/current/BP-125853089-172.17.0.2-1734243127551/current, will proceed with Du for space computation calculation, 2024-12-15T06:12:12,985 WARN [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data1/current/BP-125853089-172.17.0.2-1734243127551/current, will proceed with Du for space computation calculation, 2024-12-15T06:12:13,043 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:12:13,044 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:12:13,119 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaad0b017e635d564 with lease ID 0x6bf0f5bf7a0c334d: Processing first storage report for DS-08d47db9-2658-43ed-b76f-1828509c5271 from datanode DatanodeRegistration(127.0.0.1:34455, datanodeUuid=edc7e84d-76f0-4523-bec7-98ce6c94b361, infoPort=35005, infoSecurePort=0, ipcPort=36655, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551) 2024-12-15T06:12:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaad0b017e635d564 with lease ID 0x6bf0f5bf7a0c334d: from storage DS-08d47db9-2658-43ed-b76f-1828509c5271 node DatanodeRegistration(127.0.0.1:34455, datanodeUuid=edc7e84d-76f0-4523-bec7-98ce6c94b361, infoPort=35005, infoSecurePort=0, ipcPort=36655, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-15T06:12:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1905344b9388840 with lease ID 0x6bf0f5bf7a0c334e: Processing first storage report for DS-5fb65a3d-836c-4535-babe-b7d4b8291240 from datanode DatanodeRegistration(127.0.0.1:41211, datanodeUuid=1adafe78-888a-4d0a-84fc-3779564145a9, infoPort=37479, infoSecurePort=0, ipcPort=34495, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551) 2024-12-15T06:12:13,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1905344b9388840 with lease ID 0x6bf0f5bf7a0c334e: from storage DS-5fb65a3d-836c-4535-babe-b7d4b8291240 node DatanodeRegistration(127.0.0.1:41211, datanodeUuid=1adafe78-888a-4d0a-84fc-3779564145a9, infoPort=37479, infoSecurePort=0, ipcPort=34495, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:12:13,122 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaad0b017e635d564 with lease ID 0x6bf0f5bf7a0c334d: Processing first storage report for DS-bcebc32c-f8e5-4c9e-a447-954a7eaa2c16 from datanode DatanodeRegistration(127.0.0.1:34455, datanodeUuid=edc7e84d-76f0-4523-bec7-98ce6c94b361, infoPort=35005, infoSecurePort=0, ipcPort=36655, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551) 2024-12-15T06:12:13,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaad0b017e635d564 with lease ID 0x6bf0f5bf7a0c334d: from storage DS-bcebc32c-f8e5-4c9e-a447-954a7eaa2c16 node DatanodeRegistration(127.0.0.1:34455, datanodeUuid=edc7e84d-76f0-4523-bec7-98ce6c94b361, infoPort=35005, infoSecurePort=0, ipcPort=36655, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:12:13,123 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1905344b9388840 with lease ID 0x6bf0f5bf7a0c334e: Processing first storage report for DS-cb5d0186-351e-4311-b95d-3df48898ce01 from datanode DatanodeRegistration(127.0.0.1:41211, datanodeUuid=1adafe78-888a-4d0a-84fc-3779564145a9, infoPort=37479, infoSecurePort=0, ipcPort=34495, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551) 2024-12-15T06:12:13,123 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1905344b9388840 with lease ID 0x6bf0f5bf7a0c334e: from storage DS-cb5d0186-351e-4311-b95d-3df48898ce01 node DatanodeRegistration(127.0.0.1:41211, datanodeUuid=1adafe78-888a-4d0a-84fc-3779564145a9, infoPort=37479, infoSecurePort=0, ipcPort=34495, storageInfo=lv=-57;cid=testClusterID;nsid=1458398851;c=1734243127551), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-15T06:12:13,180 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39 2024-12-15T06:12:13,280 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/zookeeper_0, clientPort=59686, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-15T06:12:13,291 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=59686 2024-12-15T06:12:13,303 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:13,306 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:13,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:12:13,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:12:14,057 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659 with version=8 2024-12-15T06:12:14,058 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase-staging 2024-12-15T06:12:14,194 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-15T06:12:14,539 INFO [Time-limited test {}] client.ConnectionUtils(129): master/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:12:14,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:12:14,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:12:14,567 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:12:14,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:12:14,568 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:12:14,830 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:12:14,950 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-15T06:12:14,975 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-15T06:12:14,981 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:12:15,054 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24178 (auto-detected) 2024-12-15T06:12:15,055 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-15T06:12:15,131 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40589 2024-12-15T06:12:15,144 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:15,149 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:15,168 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40589 connecting to ZooKeeper ensemble=127.0.0.1:59686 2024-12-15T06:12:15,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405890x0, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:12:15,261 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40589-0x100944de0320000 connected 2024-12-15T06:12:15,410 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:12:15,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:12:15,416 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:12:15,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40589 2024-12-15T06:12:15,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40589 2024-12-15T06:12:15,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40589 2024-12-15T06:12:15,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40589 2024-12-15T06:12:15,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40589 2024-12-15T06:12:15,493 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659, hbase.cluster.distributed=false 2024-12-15T06:12:15,605 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:12:15,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:12:15,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:12:15,606 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:12:15,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:12:15,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:12:15,610 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:12:15,613 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:12:15,614 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42519 2024-12-15T06:12:15,616 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:12:15,622 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:12:15,623 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:15,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:15,632 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:42519 connecting to ZooKeeper ensemble=127.0.0.1:59686 2024-12-15T06:12:15,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425190x0, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:12:15,637 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425190x0, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:12:15,637 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42519-0x100944de0320001 connected 2024-12-15T06:12:15,639 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:12:15,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:12:15,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42519 2024-12-15T06:12:15,642 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42519 2024-12-15T06:12:15,642 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42519 2024-12-15T06:12:15,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42519 2024-12-15T06:12:15,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42519 2024-12-15T06:12:15,651 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/00a5f8d66132,40589,1734243134188 2024-12-15T06:12:15,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:12:15,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:12:15,660 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00a5f8d66132,40589,1734243134188 2024-12-15T06:12:15,668 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00a5f8d66132:40589 2024-12-15T06:12:15,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:12:15,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:12:15,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:15,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:15,682 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:12:15,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:12:15,683 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00a5f8d66132,40589,1734243134188 from backup master directory 2024-12-15T06:12:15,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00a5f8d66132,40589,1734243134188 2024-12-15T06:12:15,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:12:15,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:12:15,687 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:12:15,687 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00a5f8d66132,40589,1734243134188 2024-12-15T06:12:15,690 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-15T06:12:15,692 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-15T06:12:15,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:12:15,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:12:15,771 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase.id with ID: db318d88-905e-4c24-8bc1-383317bcc0b1 2024-12-15T06:12:15,810 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:15,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:15,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:15,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:12:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:12:15,871 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:12:15,873 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-15T06:12:15,879 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:12:15,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:12:15,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:12:15,958 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store 2024-12-15T06:12:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:12:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:12:16,414 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-15T06:12:16,417 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:16,420 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:12:16,421 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:12:16,421 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:12:16,421 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:12:16,422 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:12:16,422 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:12:16,422 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:12:16,432 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/.initializing 2024-12-15T06:12:16,432 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/WALs/00a5f8d66132,40589,1734243134188 2024-12-15T06:12:16,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C40589%2C1734243134188, suffix=, logDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/WALs/00a5f8d66132,40589,1734243134188, archiveDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/oldWALs, maxLogs=10 2024-12-15T06:12:16,556 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C40589%2C1734243134188.1734243136540 2024-12-15T06:12:16,557 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag. 2024-12-15T06:12:16,558 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag. 2024-12-15T06:12:16,612 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/WALs/00a5f8d66132,40589,1734243134188/00a5f8d66132%2C40589%2C1734243134188.1734243136540 2024-12-15T06:12:16,624 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:12:16,625 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:12:16,626 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:16,630 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,632 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-15T06:12:16,731 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:16,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:16,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-15T06:12:16,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:16,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:12:16,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,761 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-15T06:12:16,762 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:16,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:12:16,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,779 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-15T06:12:16,779 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:16,782 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:12:16,789 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,796 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,837 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-15T06:12:16,866 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:12:16,902 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:12:16,904 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=693103, jitterRate=-0.11867466568946838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-15T06:12:16,911 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:12:16,918 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-15T06:12:16,992 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a62c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:12:17,063 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-15T06:12:17,086 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-15T06:12:17,086 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-15T06:12:17,094 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-15T06:12:17,104 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 11 msec 2024-12-15T06:12:17,110 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 5 msec 2024-12-15T06:12:17,110 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-15T06:12:17,168 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-15T06:12:17,211 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-15T06:12:17,224 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-15T06:12:17,229 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-15T06:12:17,233 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-15T06:12:17,234 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-15T06:12:17,237 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-15T06:12:17,252 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-15T06:12:17,261 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-15T06:12:17,263 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-15T06:12:17,272 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-15T06:12:17,287 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-15T06:12:17,290 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-15T06:12:17,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:12:17,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:17,306 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=00a5f8d66132,40589,1734243134188, sessionid=0x100944de0320000, setting cluster-up flag (Was=false) 2024-12-15T06:12:17,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:12:17,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:17,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:17,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:17,368 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-15T06:12:17,372 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,40589,1734243134188 2024-12-15T06:12:17,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:17,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:17,418 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-15T06:12:17,430 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,40589,1734243134188 2024-12-15T06:12:17,501 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00a5f8d66132:42519 2024-12-15T06:12:17,515 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1008): ClusterId : db318d88-905e-4c24-8bc1-383317bcc0b1 2024-12-15T06:12:17,529 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:12:17,550 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:12:17,551 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:12:17,555 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:12:17,561 DEBUG [RS:0;00a5f8d66132:42519 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f38d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:12:17,564 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-15T06:12:17,570 DEBUG [RS:0;00a5f8d66132:42519 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2a5934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:12:17,573 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-15T06:12:17,574 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:12:17,574 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:12:17,574 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:12:17,578 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-15T06:12:17,578 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,40589,1734243134188 with isa=00a5f8d66132/172.17.0.2:42519, startcode=1734243135604 2024-12-15T06:12:17,589 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00a5f8d66132,40589,1734243134188 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-15T06:12:17,594 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:12:17,595 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:12:17,596 DEBUG [RS:0;00a5f8d66132:42519 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:12:17,595 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:12:17,598 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:12:17,598 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00a5f8d66132:0, corePoolSize=10, maxPoolSize=10 2024-12-15T06:12:17,598 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:17,599 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:12:17,599 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:17,649 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734243167649 2024-12-15T06:12:17,651 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-15T06:12:17,653 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-15T06:12:17,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-15T06:12:17,662 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-15T06:12:17,662 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-15T06:12:17,657 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:12:17,667 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-15T06:12:17,672 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-15T06:12:17,680 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:17,693 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-15T06:12:17,695 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:17,699 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-15T06:12:17,699 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-15T06:12:17,696 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:12:17,705 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-15T06:12:17,706 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-15T06:12:17,709 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243137707,5,FailOnTimeoutGroup] 2024-12-15T06:12:17,714 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243137710,5,FailOnTimeoutGroup] 2024-12-15T06:12:17,714 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:17,714 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-15T06:12:17,719 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:17,719 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:17,777 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36965, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:12:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:12:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:12:17,799 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40589 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,42519,1734243135604 2024-12-15T06:12:17,802 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40589 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:17,816 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-15T06:12:17,817 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659 2024-12-15T06:12:17,832 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659 2024-12-15T06:12:17,832 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35345 2024-12-15T06:12:17,832 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:12:17,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:12:17,856 DEBUG [RS:0;00a5f8d66132:42519 {}] zookeeper.ZKUtil(111): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,42519,1734243135604 2024-12-15T06:12:17,856 WARN [RS:0;00a5f8d66132:42519 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:12:17,857 INFO [RS:0;00a5f8d66132:42519 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:12:17,857 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604 2024-12-15T06:12:17,868 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,42519,1734243135604] 2024-12-15T06:12:17,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:12:17,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:12:17,916 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:12:17,939 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:12:17,974 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:12:17,991 INFO [RS:0;00a5f8d66132:42519 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:12:17,992 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:17,999 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:12:18,019 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:18,019 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,019 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,020 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,020 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,020 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,021 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:12:18,022 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,022 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,022 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,022 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,022 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:12:18,023 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:12:18,023 DEBUG [RS:0;00a5f8d66132:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:12:18,031 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:18,031 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:18,031 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:18,031 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:18,032 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,42519,1734243135604-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:12:18,114 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:12:18,117 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,42519,1734243135604-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:18,157 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.Replication(204): 00a5f8d66132,42519,1734243135604 started 2024-12-15T06:12:18,157 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,42519,1734243135604, RpcServer on 00a5f8d66132/172.17.0.2:42519, sessionid=0x100944de0320001 2024-12-15T06:12:18,158 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:12:18,158 DEBUG [RS:0;00a5f8d66132:42519 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,42519,1734243135604 2024-12-15T06:12:18,159 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,42519,1734243135604' 2024-12-15T06:12:18,159 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:12:18,176 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:12:18,186 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:12:18,186 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:12:18,186 DEBUG [RS:0;00a5f8d66132:42519 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,42519,1734243135604 2024-12-15T06:12:18,186 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,42519,1734243135604' 2024-12-15T06:12:18,186 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:12:18,193 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:12:18,200 DEBUG [RS:0;00a5f8d66132:42519 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:12:18,200 INFO [RS:0;00a5f8d66132:42519 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:12:18,200 INFO [RS:0;00a5f8d66132:42519 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:12:18,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:18,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:12:18,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:12:18,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:18,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:18,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:12:18,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:12:18,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:18,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:18,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:12:18,325 INFO [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C42519%2C1734243135604, suffix=, logDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604, archiveDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs, maxLogs=32 2024-12-15T06:12:18,326 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:12:18,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:18,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:18,330 INFO [RS:0;00a5f8d66132:42519 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243138330 2024-12-15T06:12:18,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740 2024-12-15T06:12:18,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740 2024-12-15T06:12:18,344 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:12:18,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:12:18,369 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:12:18,376 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=761932, jitterRate=-0.031153976917266846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:12:18,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:12:18,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:12:18,380 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:12:18,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:12:18,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:12:18,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:12:18,400 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:12:18,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:12:18,413 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:12:18,414 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-15T06:12:18,422 INFO [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243138330 2024-12-15T06:12:18,423 DEBUG [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:12:18,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-15T06:12:18,444 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-15T06:12:18,446 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-15T06:12:18,599 DEBUG [00a5f8d66132:40589 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-15T06:12:18,606 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:18,623 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,42519,1734243135604, state=OPENING 2024-12-15T06:12:18,634 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-15T06:12:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:18,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:18,639 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:12:18,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:12:18,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=00a5f8d66132,42519,1734243135604}] 2024-12-15T06:12:18,845 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,42519,1734243135604 2024-12-15T06:12:18,847 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-15T06:12:18,877 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47132, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-15T06:12:18,903 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-15T06:12:18,904 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:12:18,916 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C42519%2C1734243135604.meta, suffix=.meta, logDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604, archiveDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs, maxLogs=32 2024-12-15T06:12:18,924 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.meta.1734243138924.meta 2024-12-15T06:12:18,990 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.meta.1734243138924.meta 2024-12-15T06:12:18,990 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005),(127.0.0.1/127.0.0.1:37479:37479)] 2024-12-15T06:12:18,991 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:12:18,993 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-15T06:12:19,101 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-15T06:12:19,108 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-15T06:12:19,115 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-15T06:12:19,115 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:19,115 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-15T06:12:19,116 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-15T06:12:19,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:12:19,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:12:19,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:19,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:19,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:12:19,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:12:19,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:19,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:19,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:12:19,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:12:19,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:19,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:12:19,144 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740 2024-12-15T06:12:19,151 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740 2024-12-15T06:12:19,155 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:12:19,160 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:12:19,165 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=870926, jitterRate=0.10743981599807739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:12:19,169 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:12:19,180 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734243138836 2024-12-15T06:12:19,198 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-15T06:12:19,200 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-15T06:12:19,206 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:19,208 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,42519,1734243135604, state=OPEN 2024-12-15T06:12:19,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:12:19,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:12:19,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:12:19,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:12:19,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-15T06:12:19,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=00a5f8d66132,42519,1734243135604 in 578 msec 2024-12-15T06:12:19,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-15T06:12:19,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 822 msec 2024-12-15T06:12:19,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7850 sec 2024-12-15T06:12:19,273 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734243139273, completionTime=-1 2024-12-15T06:12:19,273 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-15T06:12:19,274 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-15T06:12:19,361 DEBUG [hconnection-0x29f42e84-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:12:19,366 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47134, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:12:19,392 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-15T06:12:19,392 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734243199392 2024-12-15T06:12:19,392 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734243259392 2024-12-15T06:12:19,393 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 119 msec 2024-12-15T06:12:19,446 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,40589,1734243134188-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:19,447 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,40589,1734243134188-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:19,447 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,40589,1734243134188-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:19,450 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00a5f8d66132:40589, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:19,461 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:19,477 DEBUG [master/00a5f8d66132:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-15T06:12:19,481 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-15T06:12:19,484 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:12:19,496 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-15T06:12:19,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:12:19,505 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:19,508 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:12:19,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:12:19,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:12:19,566 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b0a54567e43fdd76ae1f1988fcdb4cd5, NAME => 'hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659 2024-12-15T06:12:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:12:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:12:20,016 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:20,016 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing b0a54567e43fdd76ae1f1988fcdb4cd5, disabling compactions & flushes 2024-12-15T06:12:20,016 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,017 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,017 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. after waiting 0 ms 2024-12-15T06:12:20,017 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,017 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,017 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for b0a54567e43fdd76ae1f1988fcdb4cd5: 2024-12-15T06:12:20,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:12:20,050 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734243140034"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243140034"}]},"ts":"1734243140034"} 2024-12-15T06:12:20,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:12:20,098 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:12:20,102 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243140098"}]},"ts":"1734243140098"} 2024-12-15T06:12:20,109 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-15T06:12:20,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=b0a54567e43fdd76ae1f1988fcdb4cd5, ASSIGN}] 2024-12-15T06:12:20,130 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=b0a54567e43fdd76ae1f1988fcdb4cd5, ASSIGN 2024-12-15T06:12:20,132 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=b0a54567e43fdd76ae1f1988fcdb4cd5, ASSIGN; state=OFFLINE, location=00a5f8d66132,42519,1734243135604; forceNewPlan=false, retain=false 2024-12-15T06:12:20,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=b0a54567e43fdd76ae1f1988fcdb4cd5, regionState=OPENING, regionLocation=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:20,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure b0a54567e43fdd76ae1f1988fcdb4cd5, server=00a5f8d66132,42519,1734243135604}] 2024-12-15T06:12:20,449 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,42519,1734243135604 2024-12-15T06:12:20,480 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,480 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => b0a54567e43fdd76ae1f1988fcdb4cd5, NAME => 'hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:12:20,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:20,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,501 INFO [StoreOpener-b0a54567e43fdd76ae1f1988fcdb4cd5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,511 INFO [StoreOpener-b0a54567e43fdd76ae1f1988fcdb4cd5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0a54567e43fdd76ae1f1988fcdb4cd5 columnFamilyName info 2024-12-15T06:12:20,511 DEBUG [StoreOpener-b0a54567e43fdd76ae1f1988fcdb4cd5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:20,516 INFO [StoreOpener-b0a54567e43fdd76ae1f1988fcdb4cd5-1 {}] regionserver.HStore(327): Store=b0a54567e43fdd76ae1f1988fcdb4cd5/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:12:20,524 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,526 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:12:20,544 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:12:20,548 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened b0a54567e43fdd76ae1f1988fcdb4cd5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=691220, jitterRate=-0.12106859683990479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:12:20,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for b0a54567e43fdd76ae1f1988fcdb4cd5: 2024-12-15T06:12:20,557 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5., pid=6, masterSystemTime=1734243140449 2024-12-15T06:12:20,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,562 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:12:20,568 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=b0a54567e43fdd76ae1f1988fcdb4cd5, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:20,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-15T06:12:20,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure b0a54567e43fdd76ae1f1988fcdb4cd5, server=00a5f8d66132,42519,1734243135604 in 282 msec 2024-12-15T06:12:20,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-15T06:12:20,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=b0a54567e43fdd76ae1f1988fcdb4cd5, ASSIGN in 467 msec 2024-12-15T06:12:20,595 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:12:20,596 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243140595"}]},"ts":"1734243140595"} 2024-12-15T06:12:20,603 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-15T06:12:20,609 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-15T06:12:20,610 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:12:20,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:12:20,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:20,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.1250 sec 2024-12-15T06:12:20,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:12:20,668 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-15T06:12:20,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:12:20,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 54 msec 2024-12-15T06:12:20,729 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-15T06:12:20,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:12:20,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 26 msec 2024-12-15T06:12:20,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-15T06:12:20,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-15T06:12:20,784 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.096sec 2024-12-15T06:12:20,786 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-15T06:12:20,787 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-15T06:12:20,788 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-15T06:12:20,788 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-15T06:12:20,789 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-15T06:12:20,792 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,40589,1734243134188-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:12:20,792 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,40589,1734243134188-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-15T06:12:20,809 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-15T06:12:20,810 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-15T06:12:20,810 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,40589,1734243134188-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:12:20,844 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31e0f25a to 127.0.0.1:59686 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f9e988e 2024-12-15T06:12:20,848 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-15T06:12:20,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5e6c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:12:20,879 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-15T06:12:20,879 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-15T06:12:20,901 DEBUG [hconnection-0x5d8608a9-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:12:20,932 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47142, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:12:20,948 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=00a5f8d66132,40589,1734243134188 2024-12-15T06:12:20,948 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:12:20,966 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-15T06:12:20,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-15T06:12:21,021 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45148, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-15T06:12:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-15T06:12:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-15T06:12:21,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:12:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-12-15T06:12:21,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:12:21,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 9 2024-12-15T06:12:21,045 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:21,047 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:12:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:12:21,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741837_1013 (size=389) 2024-12-15T06:12:21,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741837_1013 (size=389) 2024-12-15T06:12:21,091 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f7ae656ca6238de57558894cccc28f57, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659 2024-12-15T06:12:21,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741838_1014 (size=72) 2024-12-15T06:12:21,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741838_1014 (size=72) 2024-12-15T06:12:21,522 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:21,522 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing f7ae656ca6238de57558894cccc28f57, disabling compactions & flushes 2024-12-15T06:12:21,522 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:21,523 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:21,523 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. after waiting 0 ms 2024-12-15T06:12:21,523 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:21,523 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:21,523 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:12:21,525 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:12:21,525 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1734243141525"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243141525"}]},"ts":"1734243141525"} 2024-12-15T06:12:21,531 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:12:21,533 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:12:21,533 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243141533"}]},"ts":"1734243141533"} 2024-12-15T06:12:21,536 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-12-15T06:12:21,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f7ae656ca6238de57558894cccc28f57, ASSIGN}] 2024-12-15T06:12:21,544 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f7ae656ca6238de57558894cccc28f57, ASSIGN 2024-12-15T06:12:21,545 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f7ae656ca6238de57558894cccc28f57, ASSIGN; state=OFFLINE, location=00a5f8d66132,42519,1734243135604; forceNewPlan=false, retain=false 2024-12-15T06:12:21,700 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=f7ae656ca6238de57558894cccc28f57, regionState=OPENING, regionLocation=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:21,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure f7ae656ca6238de57558894cccc28f57, server=00a5f8d66132,42519,1734243135604}] 2024-12-15T06:12:21,873 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,42519,1734243135604 2024-12-15T06:12:21,912 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:21,912 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => f7ae656ca6238de57558894cccc28f57, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:12:21,913 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,913 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:12:21,913 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,914 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,926 INFO [StoreOpener-f7ae656ca6238de57558894cccc28f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,938 INFO [StoreOpener-f7ae656ca6238de57558894cccc28f57-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7ae656ca6238de57558894cccc28f57 columnFamilyName info 2024-12-15T06:12:21,939 DEBUG [StoreOpener-f7ae656ca6238de57558894cccc28f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:12:21,944 INFO [StoreOpener-f7ae656ca6238de57558894cccc28f57-1 {}] regionserver.HStore(327): Store=f7ae656ca6238de57558894cccc28f57/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:12:21,952 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,953 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,968 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:21,981 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:12:21,987 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened f7ae656ca6238de57558894cccc28f57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=772899, jitterRate=-0.017208904027938843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:12:21,989 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:12:21,997 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57., pid=11, masterSystemTime=1734243141868 2024-12-15T06:12:22,016 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:22,016 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:22,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=f7ae656ca6238de57558894cccc28f57, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,42519,1734243135604 2024-12-15T06:12:22,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-15T06:12:22,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure f7ae656ca6238de57558894cccc28f57, server=00a5f8d66132,42519,1734243135604 in 330 msec 2024-12-15T06:12:22,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-15T06:12:22,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f7ae656ca6238de57558894cccc28f57, ASSIGN in 524 msec 2024-12-15T06:12:22,104 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:12:22,104 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243142104"}]},"ts":"1734243142104"} 2024-12-15T06:12:22,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-12-15T06:12:22,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:12:22,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 1.0960 sec 2024-12-15T06:12:23,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-15T06:12:23,926 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-15T06:12:23,926 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-12-15T06:12:24,632 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-15T06:12:24,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-15T06:12:24,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-15T06:12:24,923 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-15T06:12:24,924 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-15T06:12:24,925 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-15T06:12:24,925 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-15T06:12:24,926 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-15T06:12:24,927 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-15T06:12:24,927 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-15T06:12:24,927 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-15T06:12:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40589 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:12:31,062 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling, procId: 9 completed 2024-12-15T06:12:31,066 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-12-15T06:12:31,067 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:12:31,068 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243151067 2024-12-15T06:12:31,078 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243138330 with entries=4, filesize=947 B; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243151067 2024-12-15T06:12:31,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005),(127.0.0.1/127.0.0.1:37479:37479)] 2024-12-15T06:12:31,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243138330 is not closed yet, will try archiving it next time 2024-12-15T06:12:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741833_1009 (size=955) 2024-12-15T06:12:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741833_1009 (size=955) 2024-12-15T06:12:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] regionserver.HRegion(8581): Flush requested on f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:43,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f7ae656ca6238de57558894cccc28f57 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:12:43,176 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:12:43,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/0574dda8197246f38a3c5248b4265649 is 1080, key is row0001/info:/1734243151084/Put/seqid=0 2024-12-15T06:12:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741840_1016 (size=12509) 2024-12-15T06:12:43,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/0574dda8197246f38a3c5248b4265649 2024-12-15T06:12:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741840_1016 (size=12509) 2024-12-15T06:12:43,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/0574dda8197246f38a3c5248b4265649 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649 2024-12-15T06:12:43,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649, entries=7, sequenceid=11, filesize=12.2 K 2024-12-15T06:12:43,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f7ae656ca6238de57558894cccc28f57 in 209ms, sequenceid=11, compaction requested=false 2024-12-15T06:12:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:12:47,567 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:12:47,570 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54572, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:12:51,132 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243171131 2024-12-15T06:12:51,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:51,342 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243151067 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243171131 2024-12-15T06:12:51,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005),(127.0.0.1/127.0.0.1:37479:37479)] 2024-12-15T06:12:51,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243151067 is not closed yet, will try archiving it next time 2024-12-15T06:12:51,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741839_1015 (size=12399) 2024-12-15T06:12:51,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741839_1015 (size=12399) 2024-12-15T06:12:51,546 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:53,750 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:55,953 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:58,156 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] regionserver.HRegion(8581): Flush requested on f7ae656ca6238de57558894cccc28f57 2024-12-15T06:12:58,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f7ae656ca6238de57558894cccc28f57 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:12:58,358 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:58,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/caa854c8b9d24b789879c48fbf655311 is 1080, key is row0008/info:/1734243165123/Put/seqid=0 2024-12-15T06:12:58,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741842_1018 (size=12509) 2024-12-15T06:12:58,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741842_1018 (size=12509) 2024-12-15T06:12:58,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/caa854c8b9d24b789879c48fbf655311 2024-12-15T06:12:58,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/caa854c8b9d24b789879c48fbf655311 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/caa854c8b9d24b789879c48fbf655311 2024-12-15T06:12:58,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/caa854c8b9d24b789879c48fbf655311, entries=7, sequenceid=21, filesize=12.2 K 2024-12-15T06:12:58,595 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:12:58,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f7ae656ca6238de57558894cccc28f57 in 439ms, sequenceid=21, compaction requested=false 2024-12-15T06:12:58,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:12:58,596 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=24.4 K, sizeToCheck=16.0 K 2024-12-15T06:12:58,596 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:12:58,597 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649 because midkey is the same as first or last row 2024-12-15T06:13:00,360 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:00,854 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-15T06:13:00,854 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-15T06:13:02,563 WARN [sync.1 {}] wal.AbstractFSWAL(1346): Requesting log roll because we exceeded slow sync threshold; count=7, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:02,564 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C42519%2C1734243135604:(num 1734243171131) roll requested 2024-12-15T06:13:02,564 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:02,565 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243182564 2024-12-15T06:13:02,773 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:02,973 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:02,974 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243171131 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243182564 2024-12-15T06:13:02,974 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005),(127.0.0.1/127.0.0.1:37479:37479)] 2024-12-15T06:13:02,974 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243171131 is not closed yet, will try archiving it next time 2024-12-15T06:13:02,975 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243151067 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243151067 2024-12-15T06:13:02,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741841_1017 (size=7739) 2024-12-15T06:13:02,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741841_1017 (size=7739) 2024-12-15T06:13:04,766 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:06,913 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region f7ae656ca6238de57558894cccc28f57, had cached 0 bytes from a total of 25018 2024-12-15T06:13:06,969 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:09,173 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:11,377 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:13,177 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:13:13,379 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-15T06:13:13,379 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243193379 2024-12-15T06:13:18,388 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:18,388 WARN [Time-limited test {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:18,388 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C42519%2C1734243135604:(num 1734243193379) roll requested 2024-12-15T06:13:19,458 DEBUG [master/00a5f8d66132:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region b0a54567e43fdd76ae1f1988fcdb4cd5 changed from -1.0 to 0.0, refreshing cache 2024-12-15T06:13:23,388 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:23,389 WARN [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK]] 2024-12-15T06:13:23,389 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243182564 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243193379 2024-12-15T06:13:23,390 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:13:23,390 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243182564 is not closed yet, will try archiving it next time 2024-12-15T06:13:23,390 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243203390 2024-12-15T06:13:23,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741843_1019 (size=4753) 2024-12-15T06:13:23,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741843_1019 (size=4753) 2024-12-15T06:13:28,394 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:28,394 WARN [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] regionserver.HRegion(8581): Flush requested on f7ae656ca6238de57558894cccc28f57 2024-12-15T06:13:28,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f7ae656ca6238de57558894cccc28f57 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:13:28,401 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:28,401 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:30,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-15T06:13:33,396 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:33,396 WARN [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:33,402 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:33,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/223412aee0514a5185bdb5c09bba4f5f is 1080, key is row0015/info:/1734243180158/Put/seqid=0 2024-12-15T06:13:33,402 WARN [sync.0 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:33,403 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243193379 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243203390 2024-12-15T06:13:33,403 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:13:33,403 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243193379 is not closed yet, will try archiving it next time 2024-12-15T06:13:33,403 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C42519%2C1734243135604:(num 1734243203390) roll requested 2024-12-15T06:13:33,404 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243213403 2024-12-15T06:13:33,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741844_1020 (size=1569) 2024-12-15T06:13:33,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741844_1020 (size=1569) 2024-12-15T06:13:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741846_1022 (size=12509) 2024-12-15T06:13:33,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741846_1022 (size=12509) 2024-12-15T06:13:33,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/223412aee0514a5185bdb5c09bba4f5f 2024-12-15T06:13:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/223412aee0514a5185bdb5c09bba4f5f as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/223412aee0514a5185bdb5c09bba4f5f 2024-12-15T06:13:33,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/223412aee0514a5185bdb5c09bba4f5f, entries=7, sequenceid=31, filesize=12.2 K 2024-12-15T06:13:38,411 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:38,412 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:38,431 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:38,431 WARN [sync.1 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:38,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f7ae656ca6238de57558894cccc28f57 in 10037ms, sequenceid=31, compaction requested=true 2024-12-15T06:13:38,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:13:38,432 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=36.6 K, sizeToCheck=16.0 K 2024-12-15T06:13:38,432 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:13:38,432 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649 because midkey is the same as first or last row 2024-12-15T06:13:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7ae656ca6238de57558894cccc28f57:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:13:38,434 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:13:38,437 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:13:38,438 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.HStore(1540): f7ae656ca6238de57558894cccc28f57/info is initiating minor compaction (all files) 2024-12-15T06:13:38,438 INFO [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of f7ae656ca6238de57558894cccc28f57/info in TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:13:38,438 INFO [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/caa854c8b9d24b789879c48fbf655311, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/223412aee0514a5185bdb5c09bba4f5f] into tmpdir=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp, totalSize=36.6 K 2024-12-15T06:13:38,439 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0574dda8197246f38a3c5248b4265649, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1734243151084 2024-12-15T06:13:38,440 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] compactions.Compactor(224): Compacting caa854c8b9d24b789879c48fbf655311, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1734243165123 2024-12-15T06:13:38,441 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] compactions.Compactor(224): Compacting 223412aee0514a5185bdb5c09bba4f5f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1734243180158 2024-12-15T06:13:38,464 INFO [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7ae656ca6238de57558894cccc28f57#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:13:38,465 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/573f6416c562458f90516a4adf9ce909 is 1080, key is row0001/info:/1734243151084/Put/seqid=0 2024-12-15T06:13:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741848_1024 (size=27710) 2024-12-15T06:13:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741848_1024 (size=27710) 2024-12-15T06:13:38,482 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/573f6416c562458f90516a4adf9ce909 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/573f6416c562458f90516a4adf9ce909 2024-12-15T06:13:43,177 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:13:43,412 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:43,412 WARN [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:43,413 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243203390 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243213403 2024-12-15T06:13:43,413 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:13:43,413 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243203390 is not closed yet, will try archiving it next time 2024-12-15T06:13:43,414 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243171131 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243171131 2024-12-15T06:13:43,414 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C42519%2C1734243135604:(num 1734243223414) roll requested 2024-12-15T06:13:43,414 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243223414 2024-12-15T06:13:43,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741845_1021 (size=438) 2024-12-15T06:13:43,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741845_1021 (size=438) 2024-12-15T06:13:43,416 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243182564 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243182564 2024-12-15T06:13:43,418 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243193379 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243193379 2024-12-15T06:13:43,419 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243203390 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243203390 2024-12-15T06:13:48,414 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:48,414 WARN [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:48,416 INFO [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in f7ae656ca6238de57558894cccc28f57/info of f7ae656ca6238de57558894cccc28f57 into 573f6416c562458f90516a4adf9ce909(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 9sec to execute. 2024-12-15T06:13:48,416 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:13:48,416 INFO [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57., storeName=f7ae656ca6238de57558894cccc28f57/info, priority=13, startTime=1734243218433; duration=9sec 2024-12-15T06:13:48,416 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=27.1 K, sizeToCheck=16.0 K 2024-12-15T06:13:48,417 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:13:48,417 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/573f6416c562458f90516a4adf9ce909 because midkey is the same as first or last row 2024-12-15T06:13:48,417 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:13:48,417 DEBUG [RS:0;00a5f8d66132:42519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7ae656ca6238de57558894cccc28f57:info 2024-12-15T06:13:48,422 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:48,422 WARN [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-5fb65a3d-836c-4535-babe-b7d4b8291240,DISK], DatanodeInfoWithStorage[127.0.0.1:34455,DS-08d47db9-2658-43ed-b76f-1828509c5271,DISK]] 2024-12-15T06:13:48,423 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243213403 with entries=1, filesize=531 B; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243223414 2024-12-15T06:13:48,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:13:48,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243213403 is not closed yet, will try archiving it next time 2024-12-15T06:13:48,424 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243228423 2024-12-15T06:13:48,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741847_1023 (size=539) 2024-12-15T06:13:48,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741847_1023 (size=539) 2024-12-15T06:13:48,426 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243213403 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243213403 2024-12-15T06:13:48,432 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243223414 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243228423 2024-12-15T06:13:48,432 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005),(127.0.0.1/127.0.0.1:37479:37479)] 2024-12-15T06:13:48,432 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243223414 is not closed yet, will try archiving it next time 2024-12-15T06:13:48,432 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C42519%2C1734243135604:(num 1734243228423) roll requested 2024-12-15T06:13:48,432 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42519%2C1734243135604.1734243228432 2024-12-15T06:13:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741849_1025 (size=1258) 2024-12-15T06:13:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741849_1025 (size=1258) 2024-12-15T06:13:48,442 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243228423 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243228432 2024-12-15T06:13:48,442 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37479:37479),(127.0.0.1/127.0.0.1:35005:35005)] 2024-12-15T06:13:48,443 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243228423 is not closed yet, will try archiving it next time 2024-12-15T06:13:48,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741850_1026 (size=93) 2024-12-15T06:13:48,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741850_1026 (size=93) 2024-12-15T06:13:48,445 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604/00a5f8d66132%2C42519%2C1734243135604.1734243228423 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs/00a5f8d66132%2C42519%2C1734243135604.1734243228423 2024-12-15T06:13:51,914 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region f7ae656ca6238de57558894cccc28f57, had cached 0 bytes from a total of 27710 2024-12-15T06:14:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] regionserver.HRegion(8581): Flush requested on f7ae656ca6238de57558894cccc28f57 2024-12-15T06:14:00,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f7ae656ca6238de57558894cccc28f57 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:14:00,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/17903263775f4a4bbfb13471441f7020 is 1080, key is row0022/info:/1734243228425/Put/seqid=0 2024-12-15T06:14:00,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741852_1028 (size=12509) 2024-12-15T06:14:00,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741852_1028 (size=12509) 2024-12-15T06:14:00,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/17903263775f4a4bbfb13471441f7020 2024-12-15T06:14:00,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/17903263775f4a4bbfb13471441f7020 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/17903263775f4a4bbfb13471441f7020 2024-12-15T06:14:00,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/17903263775f4a4bbfb13471441f7020, entries=7, sequenceid=42, filesize=12.2 K 2024-12-15T06:14:00,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f7ae656ca6238de57558894cccc28f57 in 32ms, sequenceid=42, compaction requested=false 2024-12-15T06:14:00,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:14:00,475 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=39.3 K, sizeToCheck=16.0 K 2024-12-15T06:14:00,475 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:14:00,475 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/573f6416c562458f90516a4adf9ce909 because midkey is the same as first or last row 2024-12-15T06:14:08,451 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-15T06:14:08,452 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-15T06:14:08,452 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31e0f25a to 127.0.0.1:59686 2024-12-15T06:14:08,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:08,452 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-15T06:14:08,452 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1230360836, stopped=false 2024-12-15T06:14:08,453 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=00a5f8d66132,40589,1734243134188 2024-12-15T06:14:08,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:08,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:08,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-15T06:14:08,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:08,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:08,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:08,457 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,42519,1734243135604' ***** 2024-12-15T06:14:08,457 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:14:08,457 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:14:08,457 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:08,458 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:08,458 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:14:08,458 INFO [RS:0;00a5f8d66132:42519 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:14:08,458 INFO [RS:0;00a5f8d66132:42519 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:14:08,458 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(3579): Received CLOSE for f7ae656ca6238de57558894cccc28f57 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(3579): Received CLOSE for b0a54567e43fdd76ae1f1988fcdb4cd5 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,42519,1734243135604 2024-12-15T06:14:08,459 DEBUG [RS:0;00a5f8d66132:42519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:14:08,459 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f7ae656ca6238de57558894cccc28f57, disabling compactions & flushes 2024-12-15T06:14:08,459 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:14:08,459 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:14:08,459 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. after waiting 0 ms 2024-12-15T06:14:08,459 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:14:08,459 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-15T06:14:08,459 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1603): Online Regions={f7ae656ca6238de57558894cccc28f57=TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57., 1588230740=hbase:meta,,1.1588230740, b0a54567e43fdd76ae1f1988fcdb4cd5=hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5.} 2024-12-15T06:14:08,459 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f7ae656ca6238de57558894cccc28f57 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-12-15T06:14:08,459 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:14:08,459 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:14:08,459 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:14:08,460 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:14:08,460 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:14:08,460 DEBUG [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, b0a54567e43fdd76ae1f1988fcdb4cd5, f7ae656ca6238de57558894cccc28f57 2024-12-15T06:14:08,460 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.81 KB heapSize=5.32 KB 2024-12-15T06:14:08,465 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/c5482d9e54ca4c30b62daf854aab35f6 is 1080, key is row0029/info:/1734243242444/Put/seqid=0 2024-12-15T06:14:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741853_1029 (size=8193) 2024-12-15T06:14:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741853_1029 (size=8193) 2024-12-15T06:14:08,473 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/c5482d9e54ca4c30b62daf854aab35f6 2024-12-15T06:14:08,482 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/.tmp/info/06a385ff3726477f98af3a8dbdc52a35 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57./info:regioninfo/1734243142020/Put/seqid=0 2024-12-15T06:14:08,483 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/.tmp/info/c5482d9e54ca4c30b62daf854aab35f6 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/c5482d9e54ca4c30b62daf854aab35f6 2024-12-15T06:14:08,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741854_1030 (size=8172) 2024-12-15T06:14:08,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741854_1030 (size=8172) 2024-12-15T06:14:08,491 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.59 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/.tmp/info/06a385ff3726477f98af3a8dbdc52a35 2024-12-15T06:14:08,491 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/c5482d9e54ca4c30b62daf854aab35f6, entries=3, sequenceid=48, filesize=8.0 K 2024-12-15T06:14:08,493 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for f7ae656ca6238de57558894cccc28f57 in 34ms, sequenceid=48, compaction requested=true 2024-12-15T06:14:08,493 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/caa854c8b9d24b789879c48fbf655311, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/223412aee0514a5185bdb5c09bba4f5f] to archive 2024-12-15T06:14:08,497 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-15T06:14:08,501 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/caa854c8b9d24b789879c48fbf655311 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/archive/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/caa854c8b9d24b789879c48fbf655311 2024-12-15T06:14:08,501 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649 to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/archive/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/0574dda8197246f38a3c5248b4265649 2024-12-15T06:14:08,502 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/223412aee0514a5185bdb5c09bba4f5f to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/archive/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/info/223412aee0514a5185bdb5c09bba4f5f 2024-12-15T06:14:08,520 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/default/TestLogRolling-testSlowSyncLogRolling/f7ae656ca6238de57558894cccc28f57/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-12-15T06:14:08,522 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/.tmp/table/081a6c1b26f14dc1b661923c1707c3cc is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1734243142104/Put/seqid=0 2024-12-15T06:14:08,523 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:14:08,523 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f7ae656ca6238de57558894cccc28f57: 2024-12-15T06:14:08,523 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1734243141030.f7ae656ca6238de57558894cccc28f57. 2024-12-15T06:14:08,523 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b0a54567e43fdd76ae1f1988fcdb4cd5, disabling compactions & flushes 2024-12-15T06:14:08,523 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:14:08,523 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:14:08,524 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. after waiting 0 ms 2024-12-15T06:14:08,524 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:14:08,524 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b0a54567e43fdd76ae1f1988fcdb4cd5 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-15T06:14:08,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741855_1031 (size=5452) 2024-12-15T06:14:08,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741855_1031 (size=5452) 2024-12-15T06:14:08,529 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=232 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/.tmp/table/081a6c1b26f14dc1b661923c1707c3cc 2024-12-15T06:14:08,536 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/.tmp/info/06a385ff3726477f98af3a8dbdc52a35 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/info/06a385ff3726477f98af3a8dbdc52a35 2024-12-15T06:14:08,543 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/info/06a385ff3726477f98af3a8dbdc52a35, entries=20, sequenceid=14, filesize=8.0 K 2024-12-15T06:14:08,545 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/.tmp/table/081a6c1b26f14dc1b661923c1707c3cc as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/table/081a6c1b26f14dc1b661923c1707c3cc 2024-12-15T06:14:08,545 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/.tmp/info/4bfca9888fe642b4a6aba308ac1abebc is 45, key is default/info:d/1734243140687/Put/seqid=0 2024-12-15T06:14:08,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741856_1032 (size=5037) 2024-12-15T06:14:08,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741856_1032 (size=5037) 2024-12-15T06:14:08,553 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/table/081a6c1b26f14dc1b661923c1707c3cc, entries=4, sequenceid=14, filesize=5.3 K 2024-12-15T06:14:08,553 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/.tmp/info/4bfca9888fe642b4a6aba308ac1abebc 2024-12-15T06:14:08,554 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.81 KB/2882, heapSize ~5.04 KB/5160, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=14, compaction requested=false 2024-12-15T06:14:08,559 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1 2024-12-15T06:14:08,560 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-15T06:14:08,560 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:14:08,561 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:14:08,561 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/.tmp/info/4bfca9888fe642b4a6aba308ac1abebc as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/info/4bfca9888fe642b4a6aba308ac1abebc 2024-12-15T06:14:08,561 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-15T06:14:08,568 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/info/4bfca9888fe642b4a6aba308ac1abebc, entries=2, sequenceid=6, filesize=4.9 K 2024-12-15T06:14:08,569 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for b0a54567e43fdd76ae1f1988fcdb4cd5 in 45ms, sequenceid=6, compaction requested=false 2024-12-15T06:14:08,573 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/data/hbase/namespace/b0a54567e43fdd76ae1f1988fcdb4cd5/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-15T06:14:08,574 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:14:08,574 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b0a54567e43fdd76ae1f1988fcdb4cd5: 2024-12-15T06:14:08,574 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734243139482.b0a54567e43fdd76ae1f1988fcdb4cd5. 2024-12-15T06:14:08,660 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,42519,1734243135604; all regions closed. 2024-12-15T06:14:08,661 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604 2024-12-15T06:14:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741834_1010 (size=4330) 2024-12-15T06:14:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741834_1010 (size=4330) 2024-12-15T06:14:08,666 DEBUG [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs 2024-12-15T06:14:08,667 INFO [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C42519%2C1734243135604.meta:.meta(num 1734243138924) 2024-12-15T06:14:08,667 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/WALs/00a5f8d66132,42519,1734243135604 2024-12-15T06:14:08,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741851_1027 (size=13066) 2024-12-15T06:14:08,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741851_1027 (size=13066) 2024-12-15T06:14:08,673 DEBUG [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/oldWALs 2024-12-15T06:14:08,673 INFO [RS:0;00a5f8d66132:42519 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C42519%2C1734243135604:(num 1734243228432) 2024-12-15T06:14:08,673 DEBUG [RS:0;00a5f8d66132:42519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:08,673 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:14:08,674 INFO [RS:0;00a5f8d66132:42519 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-15T06:14:08,674 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:14:08,674 INFO [RS:0;00a5f8d66132:42519 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42519 2024-12-15T06:14:08,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,42519,1734243135604 2024-12-15T06:14:08,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:14:08,680 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,42519,1734243135604] 2024-12-15T06:14:08,680 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,42519,1734243135604; numProcessing=1 2024-12-15T06:14:08,682 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,42519,1734243135604 already deleted, retry=false 2024-12-15T06:14:08,682 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,42519,1734243135604 expired; onlineServers=0 2024-12-15T06:14:08,682 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,40589,1734243134188' ***** 2024-12-15T06:14:08,682 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-15T06:14:08,682 DEBUG [M:0;00a5f8d66132:40589 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7fdd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:14:08,682 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,40589,1734243134188 2024-12-15T06:14:08,682 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,40589,1734243134188; all regions closed. 2024-12-15T06:14:08,682 DEBUG [M:0;00a5f8d66132:40589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:08,682 DEBUG [M:0;00a5f8d66132:40589 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-15T06:14:08,683 DEBUG [M:0;00a5f8d66132:40589 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-15T06:14:08,683 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-15T06:14:08,683 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243137710 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243137710,5,FailOnTimeoutGroup] 2024-12-15T06:14:08,683 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243137707 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243137707,5,FailOnTimeoutGroup] 2024-12-15T06:14:08,683 INFO [M:0;00a5f8d66132:40589 {}] hbase.ChoreService(370): Chore service for: master/00a5f8d66132:0 had [] on shutdown 2024-12-15T06:14:08,683 DEBUG [M:0;00a5f8d66132:40589 {}] master.HMaster(1733): Stopping service threads 2024-12-15T06:14:08,683 INFO [M:0;00a5f8d66132:40589 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-15T06:14:08,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-15T06:14:08,684 INFO [M:0;00a5f8d66132:40589 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-15T06:14:08,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:08,684 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-15T06:14:08,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:14:08,684 DEBUG [M:0;00a5f8d66132:40589 {}] zookeeper.ZKUtil(347): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-15T06:14:08,684 WARN [M:0;00a5f8d66132:40589 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-15T06:14:08,685 INFO [M:0;00a5f8d66132:40589 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-15T06:14:08,685 INFO [M:0;00a5f8d66132:40589 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-15T06:14:08,685 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:14:08,685 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:08,685 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:08,685 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:14:08,685 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:08,685 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.21 KB heapSize=50.14 KB 2024-12-15T06:14:08,704 DEBUG [M:0;00a5f8d66132:40589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/de62b9ffc55a42b59acdcc7c1a80de93 is 82, key is hbase:meta,,1/info:regioninfo/1734243139205/Put/seqid=0 2024-12-15T06:14:08,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741857_1033 (size=5672) 2024-12-15T06:14:08,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741857_1033 (size=5672) 2024-12-15T06:14:08,711 INFO [M:0;00a5f8d66132:40589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/de62b9ffc55a42b59acdcc7c1a80de93 2024-12-15T06:14:08,734 DEBUG [M:0;00a5f8d66132:40589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2402b3c822dd44f695220f62ffd2d714 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734243142135/Put/seqid=0 2024-12-15T06:14:08,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741858_1034 (size=6426) 2024-12-15T06:14:08,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741858_1034 (size=6426) 2024-12-15T06:14:08,741 INFO [M:0;00a5f8d66132:40589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.61 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2402b3c822dd44f695220f62ffd2d714 2024-12-15T06:14:08,748 INFO [M:0;00a5f8d66132:40589 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2402b3c822dd44f695220f62ffd2d714 2024-12-15T06:14:08,764 DEBUG [M:0;00a5f8d66132:40589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa461decc2db4d1086721f0437a3d1ac is 69, key is 00a5f8d66132,42519,1734243135604/rs:state/1734243137810/Put/seqid=0 2024-12-15T06:14:08,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741859_1035 (size=5156) 2024-12-15T06:14:08,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741859_1035 (size=5156) 2024-12-15T06:14:08,770 INFO [M:0;00a5f8d66132:40589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa461decc2db4d1086721f0437a3d1ac 2024-12-15T06:14:08,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:14:08,780 INFO [RS:0;00a5f8d66132:42519 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,42519,1734243135604; zookeeper connection closed. 2024-12-15T06:14:08,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x100944de0320001, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:14:08,782 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@60b7e313 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@60b7e313 2024-12-15T06:14:08,783 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-15T06:14:08,793 DEBUG [M:0;00a5f8d66132:40589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e8798a994d0541628631a89b4f950df4 is 52, key is load_balancer_on/state:d/1734243140956/Put/seqid=0 2024-12-15T06:14:08,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741860_1036 (size=5056) 2024-12-15T06:14:08,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741860_1036 (size=5056) 2024-12-15T06:14:08,799 INFO [M:0;00a5f8d66132:40589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e8798a994d0541628631a89b4f950df4 2024-12-15T06:14:08,807 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/de62b9ffc55a42b59acdcc7c1a80de93 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/de62b9ffc55a42b59acdcc7c1a80de93 2024-12-15T06:14:08,813 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/de62b9ffc55a42b59acdcc7c1a80de93, entries=8, sequenceid=104, filesize=5.5 K 2024-12-15T06:14:08,814 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2402b3c822dd44f695220f62ffd2d714 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2402b3c822dd44f695220f62ffd2d714 2024-12-15T06:14:08,820 INFO [M:0;00a5f8d66132:40589 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2402b3c822dd44f695220f62ffd2d714 2024-12-15T06:14:08,821 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2402b3c822dd44f695220f62ffd2d714, entries=11, sequenceid=104, filesize=6.3 K 2024-12-15T06:14:08,822 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa461decc2db4d1086721f0437a3d1ac as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aa461decc2db4d1086721f0437a3d1ac 2024-12-15T06:14:08,828 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aa461decc2db4d1086721f0437a3d1ac, entries=1, sequenceid=104, filesize=5.0 K 2024-12-15T06:14:08,829 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e8798a994d0541628631a89b4f950df4 as hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e8798a994d0541628631a89b4f950df4 2024-12-15T06:14:08,835 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e8798a994d0541628631a89b4f950df4, entries=1, sequenceid=104, filesize=4.9 K 2024-12-15T06:14:08,836 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.21 KB/41173, heapSize ~50.08 KB/51280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=104, compaction requested=false 2024-12-15T06:14:08,838 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:08,839 DEBUG [M:0;00a5f8d66132:40589 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:14:08,839 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/MasterData/WALs/00a5f8d66132,40589,1734243134188 2024-12-15T06:14:08,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34455 is added to blk_1073741830_1006 (size=48474) 2024-12-15T06:14:08,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741830_1006 (size=48474) 2024-12-15T06:14:08,842 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:14:08,842 INFO [M:0;00a5f8d66132:40589 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-15T06:14:08,842 INFO [M:0;00a5f8d66132:40589 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40589 2024-12-15T06:14:08,844 DEBUG [M:0;00a5f8d66132:40589 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/00a5f8d66132,40589,1734243134188 already deleted, retry=false 2024-12-15T06:14:08,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:14:08,946 INFO [M:0;00a5f8d66132:40589 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,40589,1734243134188; zookeeper connection closed. 2024-12-15T06:14:08,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40589-0x100944de0320000, quorum=127.0.0.1:59686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:14:08,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42348d05{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:08,953 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53406647{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:08,953 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:08,954 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5782a133{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:08,954 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a80aaee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:08,957 WARN [BP-125853089-172.17.0.2-1734243127551 heartbeating to localhost/127.0.0.1:35345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:14:08,957 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:14:08,957 WARN [BP-125853089-172.17.0.2-1734243127551 heartbeating to localhost/127.0.0.1:35345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125853089-172.17.0.2-1734243127551 (Datanode Uuid edc7e84d-76f0-4523-bec7-98ce6c94b361) service to localhost/127.0.0.1:35345 2024-12-15T06:14:08,957 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:14:08,958 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data3/current/BP-125853089-172.17.0.2-1734243127551 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:08,959 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data4/current/BP-125853089-172.17.0.2-1734243127551 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:08,959 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:14:08,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6170d8a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:08,962 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b349526{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:08,962 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:08,962 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53ad73b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:08,962 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16f0f758{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:08,963 WARN [BP-125853089-172.17.0.2-1734243127551 heartbeating to localhost/127.0.0.1:35345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:14:08,963 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:14:08,963 WARN [BP-125853089-172.17.0.2-1734243127551 heartbeating to localhost/127.0.0.1:35345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125853089-172.17.0.2-1734243127551 (Datanode Uuid 1adafe78-888a-4d0a-84fc-3779564145a9) service to localhost/127.0.0.1:35345 2024-12-15T06:14:08,963 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:14:08,964 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data1/current/BP-125853089-172.17.0.2-1734243127551 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:08,964 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/cluster_6324d878-25dd-9ac4-715b-352efa8bc41a/dfs/data/data2/current/BP-125853089-172.17.0.2-1734243127551 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:08,965 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:14:08,973 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@dfacfed{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:14:08,974 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6483b5b2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:08,974 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:08,974 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65cdbba1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:08,974 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1342c19c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:08,983 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-15T06:14:09,019 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-15T06:14:09,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=65 (was 12) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35345 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/00a5f8d66132:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35345 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RS-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/00a5f8d66132:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/00a5f8d66132:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:35345 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:35345 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:35345 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35345 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/00a5f8d66132:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@6d9d3d71 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35345 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:35345 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=403 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 656), ProcessCount=11 (was 11), AvailableMemoryMB=4835 (was 3285) - AvailableMemoryMB LEAK? - 2024-12-15T06:14:09,033 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=66, OpenFileDescriptor=403, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=4835 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.log.dir so I do NOT create it in target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9412501c-04ed-f785-0a01-055dc5022c39/hadoop.tmp.dir so I do NOT create it in target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed, deleteOnExit=true 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/test.cache.data in system properties and HBase conf 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.tmp.dir in system properties and HBase conf 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir in system properties and HBase conf 2024-12-15T06:14:09,034 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-15T06:14:09,035 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:14:09,035 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:14:09,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-15T06:14:09,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/nfs.dump.dir in system properties and HBase conf 2024-12-15T06:14:09,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir in system properties and HBase conf 2024-12-15T06:14:09,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:14:09,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-15T06:14:09,036 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-15T06:14:09,050 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:14:09,118 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:09,123 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:09,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:09,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:09,125 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:14:09,126 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:09,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a305303{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:09,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c5b422f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:09,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9614fc0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-43745-hadoop-hdfs-3_4_1-tests_jar-_-any-11572895231046866239/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:14:09,243 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4403aa3{HTTP/1.1, (http/1.1)}{localhost:43745} 2024-12-15T06:14:09,243 INFO [Time-limited test {}] server.Server(415): Started @126372ms 2024-12-15T06:14:09,257 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:14:09,321 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:09,325 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:09,326 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:09,326 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:09,326 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:14:09,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31489321{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:09,327 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e9702f6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:09,442 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43ac091c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-46583-hadoop-hdfs-3_4_1-tests_jar-_-any-14343039401376359194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:09,443 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16593f64{HTTP/1.1, (http/1.1)}{localhost:46583} 2024-12-15T06:14:09,443 INFO [Time-limited test {}] server.Server(415): Started @126572ms 2024-12-15T06:14:09,444 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:14:09,477 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:09,481 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:09,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:09,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:09,482 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:14:09,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3eb142a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:09,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ff4fb3e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:09,532 WARN [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data1/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:09,532 WARN [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data2/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:09,552 WARN [Thread-431 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:14:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd474ba04b96e0cf4 with lease ID 0x74abb27800b4340c: Processing first storage report for DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24 from datanode DatanodeRegistration(127.0.0.1:42215, datanodeUuid=801ebdc3-2a16-4345-b707-fd72ff603e83, infoPort=46801, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd474ba04b96e0cf4 with lease ID 0x74abb27800b4340c: from storage DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24 node DatanodeRegistration(127.0.0.1:42215, datanodeUuid=801ebdc3-2a16-4345-b707-fd72ff603e83, infoPort=46801, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-15T06:14:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd474ba04b96e0cf4 with lease ID 0x74abb27800b4340c: Processing first storage report for DS-9bfc95a3-1733-4289-a376-ddae72f36716 from datanode DatanodeRegistration(127.0.0.1:42215, datanodeUuid=801ebdc3-2a16-4345-b707-fd72ff603e83, infoPort=46801, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd474ba04b96e0cf4 with lease ID 0x74abb27800b4340c: from storage DS-9bfc95a3-1733-4289-a376-ddae72f36716 node DatanodeRegistration(127.0.0.1:42215, datanodeUuid=801ebdc3-2a16-4345-b707-fd72ff603e83, infoPort=46801, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:09,603 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1eea8cb9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-34701-hadoop-hdfs-3_4_1-tests_jar-_-any-8526739286844518135/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:09,603 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4009d93a{HTTP/1.1, (http/1.1)}{localhost:34701} 2024-12-15T06:14:09,603 INFO [Time-limited test {}] server.Server(415): Started @126732ms 2024-12-15T06:14:09,605 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:14:09,690 WARN [Thread-478 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data3/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:09,690 WARN [Thread-479 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data4/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:09,709 WARN [Thread-467 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:14:09,712 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a694b09ce79f46c with lease ID 0x74abb27800b4340d: Processing first storage report for DS-ae2277ff-2795-4fb0-bebf-73e886f57a05 from datanode DatanodeRegistration(127.0.0.1:38217, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=46447, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:09,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a694b09ce79f46c with lease ID 0x74abb27800b4340d: from storage DS-ae2277ff-2795-4fb0-bebf-73e886f57a05 node DatanodeRegistration(127.0.0.1:38217, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=46447, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:09,713 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a694b09ce79f46c with lease ID 0x74abb27800b4340d: Processing first storage report for DS-51624e6b-937c-4cb1-b243-fae5b83dae7b from datanode DatanodeRegistration(127.0.0.1:38217, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=46447, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:09,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a694b09ce79f46c with lease ID 0x74abb27800b4340d: from storage DS-51624e6b-937c-4cb1-b243-fae5b83dae7b node DatanodeRegistration(127.0.0.1:38217, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=46447, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:09,735 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8 2024-12-15T06:14:09,738 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/zookeeper_0, clientPort=51726, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-15T06:14:09,739 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51726 2024-12-15T06:14:09,739 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,741 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:14:09,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:14:09,753 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b with version=8 2024-12-15T06:14:09,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase-staging 2024-12-15T06:14:09,756 INFO [Time-limited test {}] client.ConnectionUtils(129): master/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:14:09,756 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:14:09,757 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43079 2024-12-15T06:14:09,758 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,759 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,763 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:43079 connecting to ZooKeeper ensemble=127.0.0.1:51726 2024-12-15T06:14:09,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430790x0, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:14:09,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43079-0x100944fa7300000 connected 2024-12-15T06:14:09,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:14:09,787 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:09,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:14:09,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43079 2024-12-15T06:14:09,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43079 2024-12-15T06:14:09,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43079 2024-12-15T06:14:09,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43079 2024-12-15T06:14:09,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43079 2024-12-15T06:14:09,790 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b, hbase.cluster.distributed=false 2024-12-15T06:14:09,810 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:14:09,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:09,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:09,811 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:14:09,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:09,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:14:09,811 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:14:09,811 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:14:09,812 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:37129 2024-12-15T06:14:09,813 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:14:09,814 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:14:09,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,817 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,820 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37129 connecting to ZooKeeper ensemble=127.0.0.1:51726 2024-12-15T06:14:09,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371290x0, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:14:09,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37129-0x100944fa7300001 connected 2024-12-15T06:14:09,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:14:09,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:09,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:14:09,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37129 2024-12-15T06:14:09,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37129 2024-12-15T06:14:09,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37129 2024-12-15T06:14:09,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37129 2024-12-15T06:14:09,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37129 2024-12-15T06:14:09,833 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:14:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:14:09,835 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:14:09,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:14:09,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,843 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:14:09,843 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00a5f8d66132,43079,1734243249755 from backup master directory 2024-12-15T06:14:09,844 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:14:09,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:14:09,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:14:09,846 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:14:09,846 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,852 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00a5f8d66132:43079 2024-12-15T06:14:09,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:14:09,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:14:09,866 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/hbase.id with ID: 04dc899a-b8e5-48c9-840e-91f422511746 2024-12-15T06:14:09,879 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:09,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:14:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:14:09,892 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:14:09,893 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-15T06:14:09,894 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:14:09,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:14:09,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:14:09,903 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store 2024-12-15T06:14:09,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:14:09,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:14:09,911 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:09,912 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:14:09,912 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:09,912 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:09,912 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:14:09,912 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:09,912 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:14:09,912 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:14:09,913 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/.initializing 2024-12-15T06:14:09,913 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,916 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C43079%2C1734243249755, suffix=, logDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755, archiveDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/oldWALs, maxLogs=10 2024-12-15T06:14:09,917 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C43079%2C1734243249755.1734243249917 2024-12-15T06:14:09,923 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 2024-12-15T06:14:09,923 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46801:46801),(127.0.0.1/127.0.0.1:46447:46447)] 2024-12-15T06:14:09,923 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:14:09,923 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:09,924 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,924 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-15T06:14:09,927 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:09,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:09,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-15T06:14:09,930 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:09,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:14:09,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-15T06:14:09,932 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:09,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:14:09,933 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-15T06:14:09,935 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:09,936 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:14:09,937 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,937 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,940 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-15T06:14:09,941 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:14:09,944 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:14:09,945 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732371, jitterRate=-0.06874294579029083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-15T06:14:09,947 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:14:09,948 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-15T06:14:09,954 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79fa26ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:14:09,955 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-15T06:14:09,955 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-15T06:14:09,956 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-15T06:14:09,956 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-15T06:14:09,957 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-15T06:14:09,957 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-15T06:14:09,957 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-15T06:14:09,960 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-15T06:14:09,961 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-15T06:14:09,963 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-15T06:14:09,963 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-15T06:14:09,964 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-15T06:14:09,965 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-15T06:14:09,965 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-15T06:14:09,966 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-15T06:14:09,968 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-15T06:14:09,968 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-15T06:14:09,970 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-15T06:14:09,972 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-15T06:14:09,973 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-15T06:14:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,975 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=00a5f8d66132,43079,1734243249755, sessionid=0x100944fa7300000, setting cluster-up flag (Was=false) 2024-12-15T06:14:09,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,983 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-15T06:14:09,984 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:09,993 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-15T06:14:09,994 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,43079,1734243249755 2024-12-15T06:14:09,997 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-15T06:14:09,997 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-15T06:14:09,998 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00a5f8d66132,43079,1734243249755 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00a5f8d66132:0, corePoolSize=10, maxPoolSize=10 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:14:09,998 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,000 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734243280000 2024-12-15T06:14:10,000 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-15T06:14:10,000 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-15T06:14:10,000 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:14:10,000 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-15T06:14:10,000 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-15T06:14:10,000 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-15T06:14:10,000 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-15T06:14:10,001 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-15T06:14:10,001 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,001 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-15T06:14:10,002 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-15T06:14:10,002 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-15T06:14:10,002 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,002 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:14:10,007 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-15T06:14:10,007 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-15T06:14:10,007 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243250007,5,FailOnTimeoutGroup] 2024-12-15T06:14:10,011 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243250007,5,FailOnTimeoutGroup] 2024-12-15T06:14:10,011 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,011 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-15T06:14:10,011 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,011 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:14:10,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:14:10,019 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-15T06:14:10,019 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b 2024-12-15T06:14:10,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:14:10,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:14:10,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:10,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:14:10,039 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:14:10,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:10,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:14:10,047 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:14:10,047 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:10,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:14:10,049 INFO [regionserver/00a5f8d66132:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:14:10,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:14:10,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:10,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/meta/1588230740 2024-12-15T06:14:10,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/meta/1588230740 2024-12-15T06:14:10,055 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:14:10,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:14:10,058 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00a5f8d66132:37129 2024-12-15T06:14:10,060 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:14:10,060 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1008): ClusterId : 04dc899a-b8e5-48c9-840e-91f422511746 2024-12-15T06:14:10,060 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:14:10,060 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=730992, jitterRate=-0.0704968273639679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:14:10,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:14:10,062 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:14:10,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:14:10,062 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:14:10,062 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:14:10,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:14:10,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:14:10,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:14:10,063 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:14:10,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:14:10,064 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:14:10,065 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:14:10,065 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-15T06:14:10,065 DEBUG [RS:0;00a5f8d66132:37129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49915b70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:14:10,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-15T06:14:10,065 DEBUG [RS:0;00a5f8d66132:37129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5347831b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:14:10,065 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:14:10,065 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:14:10,065 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:14:10,066 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,43079,1734243249755 with isa=00a5f8d66132/172.17.0.2:37129, startcode=1734243249809 2024-12-15T06:14:10,066 DEBUG [RS:0;00a5f8d66132:37129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:14:10,067 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-15T06:14:10,068 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-15T06:14:10,073 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60517, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:14:10,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43079 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43079 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,075 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b 2024-12-15T06:14:10,076 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35953 2024-12-15T06:14:10,076 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:14:10,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:14:10,078 DEBUG [RS:0;00a5f8d66132:37129 {}] zookeeper.ZKUtil(111): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,078 WARN [RS:0;00a5f8d66132:37129 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:14:10,078 INFO [RS:0;00a5f8d66132:37129 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:14:10,078 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,078 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,37129,1734243249809] 2024-12-15T06:14:10,083 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:14:10,083 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:14:10,086 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:14:10,087 INFO [RS:0;00a5f8d66132:37129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:14:10,087 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,088 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:14:10,089 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,089 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,089 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,090 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:10,091 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:14:10,091 DEBUG [RS:0;00a5f8d66132:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:14:10,092 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,092 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,092 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,092 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,092 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,37129,1734243249809-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:14:10,115 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:14:10,115 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,37129,1734243249809-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,131 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.Replication(204): 00a5f8d66132,37129,1734243249809 started 2024-12-15T06:14:10,131 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,37129,1734243249809, RpcServer on 00a5f8d66132/172.17.0.2:37129, sessionid=0x100944fa7300001 2024-12-15T06:14:10,131 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:14:10,131 DEBUG [RS:0;00a5f8d66132:37129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,131 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,37129,1734243249809' 2024-12-15T06:14:10,131 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:14:10,132 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:14:10,132 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:14:10,132 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:14:10,132 DEBUG [RS:0;00a5f8d66132:37129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,132 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,37129,1734243249809' 2024-12-15T06:14:10,132 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:14:10,133 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:14:10,133 DEBUG [RS:0;00a5f8d66132:37129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:14:10,133 INFO [RS:0;00a5f8d66132:37129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:14:10,133 INFO [RS:0;00a5f8d66132:37129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:14:10,218 WARN [00a5f8d66132:43079 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-15T06:14:10,237 INFO [RS:0;00a5f8d66132:37129 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C37129%2C1734243249809, suffix=, logDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809, archiveDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs, maxLogs=32 2024-12-15T06:14:10,239 INFO [RS:0;00a5f8d66132:37129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.1734243250239 2024-12-15T06:14:10,246 INFO [RS:0;00a5f8d66132:37129 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 2024-12-15T06:14:10,246 DEBUG [RS:0;00a5f8d66132:37129 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46801:46801),(127.0.0.1/127.0.0.1:46447:46447)] 2024-12-15T06:14:10,468 DEBUG [00a5f8d66132:43079 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-15T06:14:10,469 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,470 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,37129,1734243249809, state=OPENING 2024-12-15T06:14:10,472 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-15T06:14:10,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:10,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:10,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=00a5f8d66132,37129,1734243249809}] 2024-12-15T06:14:10,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:14:10,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:14:10,628 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,628 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-15T06:14:10,630 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40198, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-15T06:14:10,635 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-15T06:14:10,636 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:14:10,638 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C37129%2C1734243249809.meta, suffix=.meta, logDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809, archiveDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs, maxLogs=32 2024-12-15T06:14:10,640 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta 2024-12-15T06:14:10,647 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta 2024-12-15T06:14:10,647 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46801:46801),(127.0.0.1/127.0.0.1:46447:46447)] 2024-12-15T06:14:10,647 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:14:10,648 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-15T06:14:10,648 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-15T06:14:10,648 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-15T06:14:10,648 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-15T06:14:10,648 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:10,648 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-15T06:14:10,649 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-15T06:14:10,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:14:10,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:14:10,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:10,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:14:10,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:14:10,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:10,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:14:10,655 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:14:10,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:14:10,656 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/meta/1588230740 2024-12-15T06:14:10,658 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/meta/1588230740 2024-12-15T06:14:10,659 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:14:10,661 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:14:10,662 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=730973, jitterRate=-0.07052084803581238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:14:10,663 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:14:10,664 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734243250627 2024-12-15T06:14:10,666 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-15T06:14:10,666 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-15T06:14:10,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,668 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,37129,1734243249809, state=OPEN 2024-12-15T06:14:10,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:14:10,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:14:10,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:14:10,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:14:10,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-15T06:14:10,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=00a5f8d66132,37129,1734243249809 in 198 msec 2024-12-15T06:14:10,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-15T06:14:10,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec 2024-12-15T06:14:10,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 683 msec 2024-12-15T06:14:10,681 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734243250681, completionTime=-1 2024-12-15T06:14:10,681 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-15T06:14:10,681 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-15T06:14:10,682 DEBUG [hconnection-0x32b6c180-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:14:10,684 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40200, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:14:10,685 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-15T06:14:10,685 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734243310685 2024-12-15T06:14:10,685 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734243370685 2024-12-15T06:14:10,685 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec 2024-12-15T06:14:10,691 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,43079,1734243249755-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,691 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,43079,1734243249755-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,691 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,43079,1734243249755-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,691 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00a5f8d66132:43079, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,691 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:10,692 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-15T06:14:10,692 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:14:10,693 DEBUG [master/00a5f8d66132:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-15T06:14:10,693 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-15T06:14:10,694 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:14:10,694 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:10,695 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:14:10,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:14:10,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:14:10,705 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6f9649a9ab340c5b6adc78efa277393a, NAME => 'hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b 2024-12-15T06:14:10,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:14:10,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:14:10,714 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:10,714 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 6f9649a9ab340c5b6adc78efa277393a, disabling compactions & flushes 2024-12-15T06:14:10,714 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:10,714 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:10,714 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. after waiting 0 ms 2024-12-15T06:14:10,714 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:10,714 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:10,714 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6f9649a9ab340c5b6adc78efa277393a: 2024-12-15T06:14:10,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:14:10,717 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734243250716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243250716"}]},"ts":"1734243250716"} 2024-12-15T06:14:10,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:14:10,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:14:10,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243250720"}]},"ts":"1734243250720"} 2024-12-15T06:14:10,723 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-15T06:14:10,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=6f9649a9ab340c5b6adc78efa277393a, ASSIGN}] 2024-12-15T06:14:10,728 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=6f9649a9ab340c5b6adc78efa277393a, ASSIGN 2024-12-15T06:14:10,729 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=6f9649a9ab340c5b6adc78efa277393a, ASSIGN; state=OFFLINE, location=00a5f8d66132,37129,1734243249809; forceNewPlan=false, retain=false 2024-12-15T06:14:10,880 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=6f9649a9ab340c5b6adc78efa277393a, regionState=OPENING, regionLocation=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:10,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 6f9649a9ab340c5b6adc78efa277393a, server=00a5f8d66132,37129,1734243249809}] 2024-12-15T06:14:11,036 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:11,041 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:11,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 6f9649a9ab340c5b6adc78efa277393a, NAME => 'hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:14:11,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:11,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,043 INFO [StoreOpener-6f9649a9ab340c5b6adc78efa277393a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,045 INFO [StoreOpener-6f9649a9ab340c5b6adc78efa277393a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f9649a9ab340c5b6adc78efa277393a columnFamilyName info 2024-12-15T06:14:11,045 DEBUG [StoreOpener-6f9649a9ab340c5b6adc78efa277393a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:11,046 INFO [StoreOpener-6f9649a9ab340c5b6adc78efa277393a-1 {}] regionserver.HStore(327): Store=6f9649a9ab340c5b6adc78efa277393a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:14:11,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:11,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:14:11,052 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 6f9649a9ab340c5b6adc78efa277393a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802156, jitterRate=0.019994229078292847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:14:11,053 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 6f9649a9ab340c5b6adc78efa277393a: 2024-12-15T06:14:11,054 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a., pid=6, masterSystemTime=1734243251036 2024-12-15T06:14:11,057 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:11,057 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:11,058 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=6f9649a9ab340c5b6adc78efa277393a, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:11,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-15T06:14:11,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 6f9649a9ab340c5b6adc78efa277393a, server=00a5f8d66132,37129,1734243249809 in 177 msec 2024-12-15T06:14:11,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-15T06:14:11,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=6f9649a9ab340c5b6adc78efa277393a, ASSIGN in 337 msec 2024-12-15T06:14:11,067 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:14:11,067 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243251067"}]},"ts":"1734243251067"} 2024-12-15T06:14:11,069 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-15T06:14:11,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:14:11,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 380 msec 2024-12-15T06:14:11,094 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-15T06:14:11,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:14:11,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:11,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:11,101 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-15T06:14:11,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:14:11,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 13 msec 2024-12-15T06:14:11,124 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-15T06:14:11,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:14:11,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec 2024-12-15T06:14:11,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-15T06:14:11,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.306sec 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,43079,1734243249755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:14:11,152 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,43079,1734243249755-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-15T06:14:11,154 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-15T06:14:11,155 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-15T06:14:11,155 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,43079,1734243249755-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,235 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e23eba4 to 127.0.0.1:51726 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1023b806 2024-12-15T06:14:11,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d7b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:14:11,241 DEBUG [hconnection-0x375982fe-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:14:11,243 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40216, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:14:11,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=00a5f8d66132,43079,1734243249755 2024-12-15T06:14:11,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:11,248 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-15T06:14:11,265 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:14:11,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:11,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:11,266 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:14:11,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:14:11,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:14:11,266 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:14:11,266 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:14:11,267 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42161 2024-12-15T06:14:11,267 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:14:11,267 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:14:11,268 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:11,270 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:14:11,273 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:42161 connecting to ZooKeeper ensemble=127.0.0.1:51726 2024-12-15T06:14:11,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421610x0, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:14:11,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42161-0x100944fa7300003 connected 2024-12-15T06:14:11,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:14:11,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-15T06:14:11,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:14:11,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42161 2024-12-15T06:14:11,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42161 2024-12-15T06:14:11,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42161 2024-12-15T06:14:11,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42161 2024-12-15T06:14:11,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42161 2024-12-15T06:14:11,283 DEBUG [pool-282-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-15T06:14:11,295 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;00a5f8d66132:42161 2024-12-15T06:14:11,296 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1008): ClusterId : 04dc899a-b8e5-48c9-840e-91f422511746 2024-12-15T06:14:11,296 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:14:11,299 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:14:11,299 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:14:11,301 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:14:11,301 DEBUG [RS:1;00a5f8d66132:42161 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224a7d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:14:11,302 DEBUG [RS:1;00a5f8d66132:42161 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423aa953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:14:11,302 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:14:11,302 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:14:11,302 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:14:11,303 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,43079,1734243249755 with isa=00a5f8d66132/172.17.0.2:42161, startcode=1734243251265 2024-12-15T06:14:11,303 DEBUG [RS:1;00a5f8d66132:42161 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:14:11,307 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59873, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:14:11,308 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43079 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,42161,1734243251265 2024-12-15T06:14:11,308 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43079 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,42161,1734243251265 2024-12-15T06:14:11,309 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b 2024-12-15T06:14:11,310 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35953 2024-12-15T06:14:11,310 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:14:11,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:14:11,312 DEBUG [RS:1;00a5f8d66132:42161 {}] zookeeper.ZKUtil(111): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,42161,1734243251265 2024-12-15T06:14:11,312 WARN [RS:1;00a5f8d66132:42161 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:14:11,312 INFO [RS:1;00a5f8d66132:42161 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:14:11,312 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,42161,1734243251265] 2024-12-15T06:14:11,312 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,42161,1734243251265 2024-12-15T06:14:11,319 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:14:11,319 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:14:11,322 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:14:11,322 INFO [RS:1;00a5f8d66132:42161 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:14:11,322 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,322 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:14:11,324 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:14:11,324 DEBUG [RS:1;00a5f8d66132:42161 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:14:11,326 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,326 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,326 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,326 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,326 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,42161,1734243251265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:14:11,342 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:14:11,343 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,42161,1734243251265-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:14:11,359 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.Replication(204): 00a5f8d66132,42161,1734243251265 started 2024-12-15T06:14:11,359 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,42161,1734243251265, RpcServer on 00a5f8d66132/172.17.0.2:42161, sessionid=0x100944fa7300003 2024-12-15T06:14:11,359 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:14:11,359 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3355): Started new server=Thread[RS:1;00a5f8d66132:42161,5,FailOnTimeoutGroup] 2024-12-15T06:14:11,359 DEBUG [RS:1;00a5f8d66132:42161 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,42161,1734243251265 2024-12-15T06:14:11,359 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,42161,1734243251265' 2024-12-15T06:14:11,359 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:14:11,359 INFO [Time-limited test {}] wal.TestLogRolling(191): Replication=2 2024-12-15T06:14:11,360 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:14:11,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-15T06:14:11,360 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:14:11,360 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:14:11,360 DEBUG [RS:1;00a5f8d66132:42161 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,42161,1734243251265 2024-12-15T06:14:11,360 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,42161,1734243251265' 2024-12-15T06:14:11,360 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:14:11,361 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:14:11,361 DEBUG [RS:1;00a5f8d66132:42161 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:14:11,361 INFO [RS:1;00a5f8d66132:42161 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:14:11,361 INFO [RS:1;00a5f8d66132:42161 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:14:11,363 INFO [RS-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50686, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-15T06:14:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-15T06:14:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-15T06:14:11,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:14:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-12-15T06:14:11,367 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:14:11,367 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:11,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 9 2024-12-15T06:14:11,369 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:14:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:14:11,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741837_1013 (size=393) 2024-12-15T06:14:11,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741837_1013 (size=393) 2024-12-15T06:14:11,380 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 69ff2480557a57f1cd7d9a62de1d020c, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b 2024-12-15T06:14:11,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38217 is added to blk_1073741838_1014 (size=76) 2024-12-15T06:14:11,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42215 is added to blk_1073741838_1014 (size=76) 2024-12-15T06:14:11,388 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:11,389 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1681): Closing 69ff2480557a57f1cd7d9a62de1d020c, disabling compactions & flushes 2024-12-15T06:14:11,389 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,389 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,389 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. after waiting 0 ms 2024-12-15T06:14:11,389 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,389 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,389 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1635): Region close journal for 69ff2480557a57f1cd7d9a62de1d020c: 2024-12-15T06:14:11,390 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:14:11,391 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1734243251391"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243251391"}]},"ts":"1734243251391"} 2024-12-15T06:14:11,393 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:14:11,394 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:14:11,395 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243251395"}]},"ts":"1734243251395"} 2024-12-15T06:14:11,397 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-12-15T06:14:11,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=69ff2480557a57f1cd7d9a62de1d020c, ASSIGN}] 2024-12-15T06:14:11,402 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=69ff2480557a57f1cd7d9a62de1d020c, ASSIGN 2024-12-15T06:14:11,403 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=69ff2480557a57f1cd7d9a62de1d020c, ASSIGN; state=OFFLINE, location=00a5f8d66132,37129,1734243249809; forceNewPlan=false, retain=false 2024-12-15T06:14:11,464 INFO [RS:1;00a5f8d66132:42161 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C42161%2C1734243251265, suffix=, logDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,42161,1734243251265, archiveDir=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs, maxLogs=32 2024-12-15T06:14:11,465 INFO [RS:1;00a5f8d66132:42161 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C42161%2C1734243251265.1734243251465 2024-12-15T06:14:11,473 INFO [RS:1;00a5f8d66132:42161 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,42161,1734243251265/00a5f8d66132%2C42161%2C1734243251265.1734243251465 2024-12-15T06:14:11,473 DEBUG [RS:1;00a5f8d66132:42161 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46447:46447),(127.0.0.1/127.0.0.1:46801:46801)] 2024-12-15T06:14:11,554 INFO [00a5f8d66132:43079 {}] balancer.BaseLoadBalancer(546): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-15T06:14:11,555 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=69ff2480557a57f1cd7d9a62de1d020c, regionState=OPENING, regionLocation=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:11,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 69ff2480557a57f1cd7d9a62de1d020c, server=00a5f8d66132,37129,1734243249809}] 2024-12-15T06:14:11,710 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:11,715 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,715 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 69ff2480557a57f1cd7d9a62de1d020c, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:14:11,716 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,716 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:14:11,716 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,716 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,718 INFO [StoreOpener-69ff2480557a57f1cd7d9a62de1d020c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,719 INFO [StoreOpener-69ff2480557a57f1cd7d9a62de1d020c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69ff2480557a57f1cd7d9a62de1d020c columnFamilyName info 2024-12-15T06:14:11,719 DEBUG [StoreOpener-69ff2480557a57f1cd7d9a62de1d020c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:14:11,720 INFO [StoreOpener-69ff2480557a57f1cd7d9a62de1d020c-1 {}] regionserver.HStore(327): Store=69ff2480557a57f1cd7d9a62de1d020c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:14:11,721 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,721 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,723 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:11,725 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:14:11,726 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 69ff2480557a57f1cd7d9a62de1d020c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=878420, jitterRate=0.11696889996528625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:14:11,727 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 69ff2480557a57f1cd7d9a62de1d020c: 2024-12-15T06:14:11,728 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c., pid=11, masterSystemTime=1734243251710 2024-12-15T06:14:11,730 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,730 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:11,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=69ff2480557a57f1cd7d9a62de1d020c, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,37129,1734243249809 2024-12-15T06:14:11,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-15T06:14:11,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 69ff2480557a57f1cd7d9a62de1d020c, server=00a5f8d66132,37129,1734243249809 in 176 msec 2024-12-15T06:14:11,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-15T06:14:11,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=69ff2480557a57f1cd7d9a62de1d020c, ASSIGN in 336 msec 2024-12-15T06:14:11,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:14:11,740 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243251740"}]},"ts":"1734243251740"} 2024-12-15T06:14:11,742 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-12-15T06:14:11,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:14:11,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 381 msec 2024-12-15T06:14:13,524 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:14:13,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:14:14,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:14:14,047 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:14:14,073 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:14:14,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-15T06:14:14,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-12-15T06:14:14,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-15T06:14:16,083 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-15T06:14:16,084 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-15T06:14:16,085 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-12-15T06:14:20,425 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:14:20,427 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:14:20,447 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:14:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43079 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:14:21,371 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath, procId: 9 completed 2024-12-15T06:14:21,377 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-12-15T06:14:21,377 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:21,390 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:21,394 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:21,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:21,395 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:21,395 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:14:21,395 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14dfc037{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:21,396 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aec2b87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:21,511 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19b611c1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-41889-hadoop-hdfs-3_4_1-tests_jar-_-any-13649302810365840508/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:21,511 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5338623e{HTTP/1.1, (http/1.1)}{localhost:41889} 2024-12-15T06:14:21,511 INFO [Time-limited test {}] server.Server(415): Started @138640ms 2024-12-15T06:14:21,513 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:14:21,549 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:21,553 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:21,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:21,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:21,554 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:14:21,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dd74f13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:21,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1051bc4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:21,599 WARN [Thread-634 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data5/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:21,599 WARN [Thread-635 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data6/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:21,625 WARN [Thread-614 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:14:21,628 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b5747f10f57981d with lease ID 0x74abb27800b4340e: Processing first storage report for DS-b03c15e6-e17c-4148-a059-19d67234119a from datanode DatanodeRegistration(127.0.0.1:45871, datanodeUuid=a4568fe7-572d-4cc6-93ca-3dd53622ae70, infoPort=45329, infoSecurePort=0, ipcPort=36713, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:21,629 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b5747f10f57981d with lease ID 0x74abb27800b4340e: from storage DS-b03c15e6-e17c-4148-a059-19d67234119a node DatanodeRegistration(127.0.0.1:45871, datanodeUuid=a4568fe7-572d-4cc6-93ca-3dd53622ae70, infoPort=45329, infoSecurePort=0, ipcPort=36713, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:21,629 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b5747f10f57981d with lease ID 0x74abb27800b4340e: Processing first storage report for DS-4fecbd61-f9f0-4416-96bf-7a2c7f2b8ccb from datanode DatanodeRegistration(127.0.0.1:45871, datanodeUuid=a4568fe7-572d-4cc6-93ca-3dd53622ae70, infoPort=45329, infoSecurePort=0, ipcPort=36713, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:21,629 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b5747f10f57981d with lease ID 0x74abb27800b4340e: from storage DS-4fecbd61-f9f0-4416-96bf-7a2c7f2b8ccb node DatanodeRegistration(127.0.0.1:45871, datanodeUuid=a4568fe7-572d-4cc6-93ca-3dd53622ae70, infoPort=45329, infoSecurePort=0, ipcPort=36713, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:21,679 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30f14291{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-42771-hadoop-hdfs-3_4_1-tests_jar-_-any-13761310045400136974/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:21,680 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@19dd2382{HTTP/1.1, (http/1.1)}{localhost:42771} 2024-12-15T06:14:21,680 INFO [Time-limited test {}] server.Server(415): Started @138809ms 2024-12-15T06:14:21,681 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:14:21,719 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:21,723 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:21,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:21,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:21,723 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:14:21,724 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70af60d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:21,724 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6039c610{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:21,777 WARN [Thread-669 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:21,777 WARN [Thread-670 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:21,807 WARN [Thread-649 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:14:21,810 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa765af95ce4feaa4 with lease ID 0x74abb27800b4340f: Processing first storage report for DS-936d601c-7b75-45b8-9970-f1f928b53f73 from datanode DatanodeRegistration(127.0.0.1:39877, datanodeUuid=2d779f54-bfb1-4997-b5d7-1b1579493c1a, infoPort=45847, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:21,810 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa765af95ce4feaa4 with lease ID 0x74abb27800b4340f: from storage DS-936d601c-7b75-45b8-9970-f1f928b53f73 node DatanodeRegistration(127.0.0.1:39877, datanodeUuid=2d779f54-bfb1-4997-b5d7-1b1579493c1a, infoPort=45847, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:21,810 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa765af95ce4feaa4 with lease ID 0x74abb27800b4340f: Processing first storage report for DS-c4d7bc5c-7765-45c1-b0ac-d223248a6212 from datanode DatanodeRegistration(127.0.0.1:39877, datanodeUuid=2d779f54-bfb1-4997-b5d7-1b1579493c1a, infoPort=45847, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:21,810 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa765af95ce4feaa4 with lease ID 0x74abb27800b4340f: from storage DS-c4d7bc5c-7765-45c1-b0ac-d223248a6212 node DatanodeRegistration(127.0.0.1:39877, datanodeUuid=2d779f54-bfb1-4997-b5d7-1b1579493c1a, infoPort=45847, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:21,850 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3009361f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-45801-hadoop-hdfs-3_4_1-tests_jar-_-any-11421129731437177867/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:21,850 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6ffc2fc1{HTTP/1.1, (http/1.1)}{localhost:45801} 2024-12-15T06:14:21,850 INFO [Time-limited test {}] server.Server(415): Started @138979ms 2024-12-15T06:14:21,852 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:14:21,935 WARN [Thread-695 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data9/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:21,936 WARN [Thread-696 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data10/current/BP-1324047213-172.17.0.2-1734243249069/current, will proceed with Du for space computation calculation, 2024-12-15T06:14:21,954 WARN [Thread-684 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:14:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84cce9c7bdbd2a7a with lease ID 0x74abb27800b43410: Processing first storage report for DS-dc2e1232-6a91-4577-83f4-34269edc2fd2 from datanode DatanodeRegistration(127.0.0.1:37061, datanodeUuid=7dc96e45-aa24-4580-8b53-fbbd76617730, infoPort=37187, infoSecurePort=0, ipcPort=38761, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84cce9c7bdbd2a7a with lease ID 0x74abb27800b43410: from storage DS-dc2e1232-6a91-4577-83f4-34269edc2fd2 node DatanodeRegistration(127.0.0.1:37061, datanodeUuid=7dc96e45-aa24-4580-8b53-fbbd76617730, infoPort=37187, infoSecurePort=0, ipcPort=38761, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-15T06:14:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84cce9c7bdbd2a7a with lease ID 0x74abb27800b43410: Processing first storage report for DS-88b5bf2e-c542-482b-9db3-33ee93101fde from datanode DatanodeRegistration(127.0.0.1:37061, datanodeUuid=7dc96e45-aa24-4580-8b53-fbbd76617730, infoPort=37187, infoSecurePort=0, ipcPort=38761, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069) 2024-12-15T06:14:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84cce9c7bdbd2a7a with lease ID 0x74abb27800b43410: from storage DS-88b5bf2e-c542-482b-9db3-33ee93101fde node DatanodeRegistration(127.0.0.1:37061, datanodeUuid=7dc96e45-aa24-4580-8b53-fbbd76617730, infoPort=37187, infoSecurePort=0, ipcPort=38761, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:21,978 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1eea8cb9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:21,980 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4009d93a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:21,980 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:21,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ff4fb3e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:21,975 WARN [ResponseProcessor for block BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,981 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3eb142a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:21,975 WARN [ResponseProcessor for block BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,981 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:21,975 WARN [ResponseProcessor for block BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,981 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:21,974 WARN [ResponseProcessor for block BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,982 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,42161,1734243251265/00a5f8d66132%2C42161%2C1734243251265.1734243251465 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK], DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:21,982 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta block BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:21,981 WARN [PacketResponder: BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38217] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,983 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1812173597_22 at /127.0.0.1:45968 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:38217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45968 dst: /127.0.0.1:38217 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,983 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1812173597_22 at /127.0.0.1:52494 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:42215:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52494 dst: /127.0.0.1:42215 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,983 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:52382 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42215:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52382 dst: /127.0.0.1:42215 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,983 WARN [PacketResponder: BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38217] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,984 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:45898 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45898 dst: /127.0.0.1:38217 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,984 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:52424 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42215:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52424 dst: /127.0.0.1:42215 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,985 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:45914 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45914 dst: /127.0.0.1:38217 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,985 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:14:21,985 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:14:21,985 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1324047213-172.17.0.2-1734243249069 (Datanode Uuid 8951ac71-220e-4e09-bd29-42ad8229db01) service to localhost/127.0.0.1:35953 2024-12-15T06:14:21,985 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:14:21,982 WARN [PacketResponder: BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38217] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:52426 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42215:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52426 dst: /127.0.0.1:42215 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:45930 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45930 dst: /127.0.0.1:38217 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 49756 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:21,986 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data3/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:21,986 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data4/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:21,987 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:14:21,988 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,993 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta block BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,993 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43ac091c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:21,995 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,42161,1734243251265/00a5f8d66132%2C42161%2C1734243251265.1734243251465 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741839_1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1015 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:21,996 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16593f64{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:21,996 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:21,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e9702f6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:21,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31489321{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:22,001 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:14:22,001 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1324047213-172.17.0.2-1734243249069 (Datanode Uuid 801ebdc3-2a16-4345-b707-fd72ff603e83) service to localhost/127.0.0.1:35953 2024-12-15T06:14:22,001 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:14:22,001 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:14:22,002 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data1/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:22,002 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data2/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:22,002 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:14:22,012 WARN [RS:0;00a5f8d66132:37129.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=4, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:22,012 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C37129%2C1734243249809:(num 1734243250239) roll requested 2024-12-15T06:14:22,012 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.1734243262012 2024-12-15T06:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:40216 deadline: 1734243272011, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL 2024-12-15T06:14:22,015 WARN [Thread-706 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1020 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:22,016 WARN [Thread-706 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:22,016 WARN [Thread-706 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741840_1020 2024-12-15T06:14:22,019 WARN [Thread-706 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] 2024-12-15T06:14:22,031 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL 2024-12-15T06:14:22,031 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 with entries=4, filesize=959 B; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243262012 2024-12-15T06:14:22,031 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37187:37187),(127.0.0.1/127.0.0.1:45329:45329)] 2024-12-15T06:14:22,032 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:22,032 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:22,032 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:22,033 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-15T06:14:22,033 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-15T06:14:22,033 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 2024-12-15T06:14:22,037 WARN [IPC Server handler 4 on default port 35953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741833_1009 2024-12-15T06:14:22,040 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 after 6ms 2024-12-15T06:14:26,041 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 after 4008ms 2024-12-15T06:14:34,061 INFO [Time-limited test {}] wal.TestLogRolling(243): log.getCurrentFileName(): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243262012 2024-12-15T06:14:34,062 WARN [ResponseProcessor for block BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:34,062 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243262012 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK], DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:34,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:34408 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37061:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34408 dst: /127.0.0.1:37061 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:34,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:43952 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1021] {}] datanode.DataXceiver(331): 127.0.0.1:45871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43952 dst: /127.0.0.1:45871 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:34,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3009361f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:34,065 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6ffc2fc1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:34,065 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:34,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6039c610{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:34,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70af60d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:34,067 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:14:34,068 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:14:34,068 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1324047213-172.17.0.2-1734243249069 (Datanode Uuid 7dc96e45-aa24-4580-8b53-fbbd76617730) service to localhost/127.0.0.1:35953 2024-12-15T06:14:34,068 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:14:34,068 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data9/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:34,068 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data10/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:34,069 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:14:34,071 WARN [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]] 2024-12-15T06:14:34,071 WARN [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]] 2024-12-15T06:14:34,071 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C37129%2C1734243249809:(num 1734243262012) roll requested 2024-12-15T06:14:34,072 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.1734243274072 2024-12-15T06:14:34,074 WARN [Thread-715 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:34,075 WARN [Thread-715 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:34,075 WARN [Thread-715 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741842_1024 2024-12-15T06:14:34,075 WARN [Thread-715 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:34,077 WARN [Thread-715 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:34,077 WARN [Thread-715 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]) is bad. 2024-12-15T06:14:34,077 WARN [Thread-715 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741843_1025 2024-12-15T06:14:34,078 WARN [Thread-715 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK] 2024-12-15T06:14:34,079 WARN [Thread-715 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:34,079 WARN [Thread-715 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:34,079 WARN [Thread-715 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741844_1026 2024-12-15T06:14:34,080 WARN [Thread-715 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] 2024-12-15T06:14:34,085 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243262012 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243274072 2024-12-15T06:14:34,085 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45329:45329),(127.0.0.1/127.0.0.1:45847:45847)] 2024-12-15T06:14:34,085 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:34,085 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243262012 is not closed yet, will try archiving it next time 2024-12-15T06:14:34,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45871 is added to blk_1073741841_1023 (size=2431) 2024-12-15T06:14:34,488 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:36,642 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@440f81e5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45871, datanodeUuid=a4568fe7-572d-4cc6-93ca-3dd53622ae70, infoPort=45329, infoSecurePort=0, ipcPort=36713, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741841_1023 to 127.0.0.1:38217 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,075 WARN [ResponseProcessor for block BP-1324047213-172.17.0.2-1734243249069:blk_1073741845_1027 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1324047213-172.17.0.2-1734243249069:blk_1073741845_1027 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,075 WARN [DataStreamer for file /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243274072 block BP-1324047213-172.17.0.2-1734243249069:blk_1073741845_1027 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741845_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:38,076 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42736 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741845_1027] {}] datanode.DataXceiver(331): 127.0.0.1:45871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42736 dst: /127.0.0.1:45871 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,076 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42828 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741845_1027] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42828 dst: /127.0.0.1:39877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,077 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@19b611c1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:38,078 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5338623e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:14:38,078 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:14:38,078 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aec2b87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:14:38,078 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14dfc037{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:14:38,080 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:14:38,080 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1324047213-172.17.0.2-1734243249069 (Datanode Uuid a4568fe7-572d-4cc6-93ca-3dd53622ae70) service to localhost/127.0.0.1:35953 2024-12-15T06:14:38,080 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:14:38,080 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:14:38,081 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data5/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:38,081 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data6/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:14:38,081 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:14:38,084 WARN [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]] 2024-12-15T06:14:38,084 WARN [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]] 2024-12-15T06:14:38,084 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C37129%2C1734243249809:(num 1734243274072) roll requested 2024-12-15T06:14:38,084 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.1734243278084 2024-12-15T06:14:38,087 WARN [Thread-725 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,087 WARN [Thread-725 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]) is bad. 2024-12-15T06:14:38,087 WARN [Thread-725 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741846_1029 2024-12-15T06:14:38,088 WARN [Thread-725 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK] 2024-12-15T06:14:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(8581): Flush requested on 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:38,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 69ff2480557a57f1cd7d9a62de1d020c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:14:38,090 WARN [Thread-725 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37061 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42846 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741847_1030 to mirror 127.0.0.1:37061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,091 WARN [Thread-725 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:38,091 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42846 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-15T06:14:38,091 WARN [Thread-725 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741847_1030 2024-12-15T06:14:38,091 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42846 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42846 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,092 WARN [Thread-725 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:38,096 WARN [Thread-725 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38217 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,096 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42848 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741848_1031 to mirror 127.0.0.1:38217 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,096 WARN [Thread-725 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:38,096 WARN [Thread-725 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741848_1031 2024-12-15T06:14:38,096 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42848 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-15T06:14:38,096 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42848 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42848 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,097 WARN [Thread-725 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] 2024-12-15T06:14:38,099 WARN [Thread-725 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,099 WARN [Thread-725 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:38,099 WARN [Thread-725 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741849_1032 2024-12-15T06:14:38,100 WARN [Thread-725 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:38,101 WARN [IPC Server handler 2 on default port 35953 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-15T06:14:38,101 WARN [IPC Server handler 2 on default port 35953 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-15T06:14:38,101 WARN [IPC Server handler 2 on default port 35953 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-15T06:14:38,106 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243274072 with entries=13, filesize=14.10 KB; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278084 2024-12-15T06:14:38,106 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45847:45847)] 2024-12-15T06:14:38,106 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,106 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243274072 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741845_1028 (size=14443) 2024-12-15T06:14:38,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/.tmp/info/02ca8c7b8b994233980491e36af347b7 is 1080, key is row0002/info:/1734243274070/Put/seqid=0 2024-12-15T06:14:38,113 WARN [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,113 WARN [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:38,113 WARN [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741851_1034 2024-12-15T06:14:38,114 WARN [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:38,115 WARN [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,116 WARN [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:38,116 WARN [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741852_1035 2024-12-15T06:14:38,116 WARN [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:38,118 WARN [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38217 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,118 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42864 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741853_1036 to mirror 127.0.0.1:38217 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,119 WARN [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:38,119 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42864 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-15T06:14:38,119 WARN [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741853_1036 2024-12-15T06:14:38,119 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42864 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42864 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,119 WARN [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] 2024-12-15T06:14:38,121 WARN [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42215 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,121 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42872 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741854_1037 to mirror 127.0.0.1:42215 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,121 WARN [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]) is bad. 2024-12-15T06:14:38,121 WARN [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741854_1037 2024-12-15T06:14:38,122 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42872 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-15T06:14:38,122 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42872 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42872 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,122 WARN [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK] 2024-12-15T06:14:38,122 WARN [IPC Server handler 3 on default port 35953 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-15T06:14:38,123 WARN [IPC Server handler 3 on default port 35953 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-15T06:14:38,123 WARN [IPC Server handler 3 on default port 35953 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-15T06:14:38,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741855_1038 (size=10347) 2024-12-15T06:14:38,303 WARN [sync.2 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]] 2024-12-15T06:14:38,303 WARN [sync.2 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]] 2024-12-15T06:14:38,303 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C37129%2C1734243249809:(num 1734243278084) roll requested 2024-12-15T06:14:38,303 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.1734243278303 2024-12-15T06:14:38,306 WARN [Thread-737 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,306 WARN [Thread-737 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK], DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:38,306 WARN [Thread-737 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741856_1039 2024-12-15T06:14:38,307 WARN [Thread-737 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:38,308 WARN [Thread-737 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,308 WARN [Thread-737 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]) is bad. 2024-12-15T06:14:38,308 WARN [Thread-737 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741857_1040 2024-12-15T06:14:38,309 WARN [Thread-737 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK] 2024-12-15T06:14:38,310 WARN [Thread-737 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,310 WARN [Thread-737 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK], DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]) is bad. 2024-12-15T06:14:38,310 WARN [Thread-737 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741858_1041 2024-12-15T06:14:38,311 WARN [Thread-737 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38217,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK] 2024-12-15T06:14:38,313 WARN [Thread-737 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37061 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:38,313 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42886 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741859_1042 to mirror 127.0.0.1:37061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,313 WARN [Thread-737 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:38,313 WARN [Thread-737 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741859_1042 2024-12-15T06:14:38,313 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42886 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-15T06:14:38,313 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:42886 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42886 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:38,314 WARN [Thread-737 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:38,314 WARN [IPC Server handler 4 on default port 35953 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-15T06:14:38,314 WARN [IPC Server handler 4 on default port 35953 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-15T06:14:38,314 WARN [IPC Server handler 4 on default port 35953 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-15T06:14:38,318 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278084 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278303 2024-12-15T06:14:38,318 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45847:45847)] 2024-12-15T06:14:38,318 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,318 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243274072 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,318 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278084 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741850_1033 (size=1261) 2024-12-15T06:14:38,505 WARN [sync.4 {}] wal.FSHLog(760): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-12-15T06:14:38,509 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,509 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278084 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/.tmp/info/02ca8c7b8b994233980491e36af347b7 2024-12-15T06:14:38,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/.tmp/info/02ca8c7b8b994233980491e36af347b7 as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/info/02ca8c7b8b994233980491e36af347b7 2024-12-15T06:14:38,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/info/02ca8c7b8b994233980491e36af347b7, entries=5, sequenceid=12, filesize=10.1 K 2024-12-15T06:14:38,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 69ff2480557a57f1cd7d9a62de1d020c in 455ms, sequenceid=12, compaction requested=false 2024-12-15T06:14:38,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 69ff2480557a57f1cd7d9a62de1d020c: 2024-12-15T06:14:38,715 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:14:38,719 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:14:38,719 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:14:38,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:14:38,720 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:14:38,720 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34c7ff9b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:14:38,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5697186b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:14:38,721 DEBUG [Close-WAL-Writer-2 {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 is not closed yet, will try archiving it next time 2024-12-15T06:14:38,721 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243262012 to hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs/00a5f8d66132%2C37129%2C1734243249809.1734243262012 2024-12-15T06:14:38,837 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f1d6d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/java.io.tmpdir/jetty-localhost-37243-hadoop-hdfs-3_4_1-tests_jar-_-any-1906641114027179522/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:14:38,837 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43126ed{HTTP/1.1, (http/1.1)}{localhost:37243} 2024-12-15T06:14:38,837 INFO [Time-limited test {}] server.Server(415): Started @155966ms 2024-12-15T06:14:38,839 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:14:38,933 WARN [Thread-757 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:14:38,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6879343f0b35af1 with lease ID 0x74abb27800b43411: from storage DS-ae2277ff-2795-4fb0-bebf-73e886f57a05 node DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:38,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6879343f0b35af1 with lease ID 0x74abb27800b43411: from storage DS-51624e6b-937c-4cb1-b243-fae5b83dae7b node DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:14:39,734 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:14:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741855_1038 (size=10347) 2024-12-15T06:14:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741845_1028 (size=14443) 2024-12-15T06:14:40,000 WARN [master/00a5f8d66132:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=96, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,001 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C43079%2C1734243249755:(num 1734243249917) roll requested 2024-12-15T06:14:40,001 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,002 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C43079%2C1734243249755.1734243280001 2024-12-15T06:14:40,002 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,006 WARN [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37061 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,006 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:42914 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741861_1044 to mirror 127.0.0.1:37061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:40,006 WARN [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:40,006 WARN [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741861_1044 2024-12-15T06:14:40,006 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:42914 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-15T06:14:40,007 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:42914 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42914 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:40,007 WARN [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:40,009 WARN [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45871 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,009 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:42918 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741862_1045 to mirror 127.0.0.1:45871 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:40,009 WARN [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:40,010 WARN [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741862_1045 2024-12-15T06:14:40,010 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:42918 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-15T06:14:40,010 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1843539365_22 at /127.0.0.1:42918 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42918 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:40,010 WARN [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:40,011 WARN [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,012 WARN [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK], DatanodeInfoWithStorage[127.0.0.1:38783,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]) is bad. 2024-12-15T06:14:40,012 WARN [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741863_1046 2024-12-15T06:14:40,012 WARN [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK] 2024-12-15T06:14:40,017 WARN [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL 2024-12-15T06:14:40,017 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 with entries=93, filesize=46.03 KB; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243280001 2024-12-15T06:14:40,017 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:45847:45847)] 2024-12-15T06:14:40,017 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 is not closed yet, will try archiving it next time 2024-12-15T06:14:40,017 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,018 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:40,018 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 2024-12-15T06:14:40,018 WARN [IPC Server handler 4 on default port 35953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 has not been closed. Lease recovery is in progress. RecoveryId = 1048 for block blk_1073741830_1006 2024-12-15T06:14:40,018 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 after 0ms 2024-12-15T06:14:40,810 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5fb04930[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39877, datanodeUuid=2d779f54-bfb1-4997-b5d7-1b1579493c1a, infoPort=45847, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741850_1033 to 127.0.0.1:45871 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:42,065 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:14:42,066 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45156, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:14:42,302 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:14:42,304 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45170, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:14:44,020 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755/00a5f8d66132%2C43079%2C1734243249755.1734243249917 after 4002ms 2024-12-15T06:14:45,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741838_1014 (size=76) 2024-12-15T06:14:45,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:14:46,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5388dde1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741828_1004 to 127.0.0.1:45871 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:46,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741832_1008 to 127.0.0.1:37061 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:48,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5388dde1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741827_1003 to 127.0.0.1:45871 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:48,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741829_1005 to 127.0.0.1:45871 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:48,954 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6a667fd4 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:42215,null,null]) java.net.ConnectException: Call From 00a5f8d66132/172.17.0.2 to localhost:34335 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-15T06:14:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741833_1022 (size=959) 2024-12-15T06:14:49,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741825_1001 to 127.0.0.1:45871 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:51,207 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-15T06:14:51,207 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-15T06:14:51,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741826_1002 to 127.0.0.1:45871 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741837_1013 (size=393) 2024-12-15T06:14:52,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5388dde1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741835_1011 to 127.0.0.1:37061 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:52,936 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741831_1007 to 127.0.0.1:37061 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:54,937 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38783, datanodeUuid=8951ac71-220e-4e09-bd29-42ad8229db01, infoPort=35225, infoSecurePort=0, ipcPort=41029, storageInfo=lv=-57;cid=testClusterID;nsid=1069116213;c=1734243249069):Failed to transfer BP-1324047213-172.17.0.2-1734243249069:blk_1073741833_1022 to 127.0.0.1:37061 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:56,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 69ff2480557a57f1cd7d9a62de1d020c, had cached 0 bytes from a total of 10347 2024-12-15T06:14:57,716 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.1734243297716 2024-12-15T06:14:57,723 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278303 with entries=2, filesize=1.57 KB; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243297716 2024-12-15T06:14:57,723 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:45847:45847)] 2024-12-15T06:14:57,723 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278303 is not closed yet, will try archiving it next time 2024-12-15T06:14:57,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741860_1043 (size=1618) 2024-12-15T06:14:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(8581): Flush requested on 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:57,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 69ff2480557a57f1cd7d9a62de1d020c 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-12-15T06:14:57,727 INFO [sync.3 {}] wal.FSHLog(777): LowReplication-Roller was enabled. 2024-12-15T06:14:57,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/.tmp/info/0c3fab9904814427be943f6cb1b42dab is 1080, key is row0007/info:/1734243278090/Put/seqid=0 2024-12-15T06:14:57,735 WARN [Thread-805 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1050 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,735 WARN [Thread-805 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741866_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK], DatanodeInfoWithStorage[127.0.0.1:38783,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:57,735 WARN [Thread-805 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741866_1050 2024-12-15T06:14:57,736 WARN [Thread-805 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:57,739 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-15T06:14:57,739 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-15T06:14:57,739 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4e23eba4 to 127.0.0.1:51726 2024-12-15T06:14:57,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:57,739 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-15T06:14:57,739 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1948149429, stopped=false 2024-12-15T06:14:57,739 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=00a5f8d66132,43079,1734243249755 2024-12-15T06:14:57,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:57,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:57,742 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-15T06:14:57,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:57,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:57,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:57,742 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,37129,1734243249809' ***** 2024-12-15T06:14:57,742 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:14:57,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741867_1051 (size=13583) 2024-12-15T06:14:57,742 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,42161,1734243251265' ***** 2024-12-15T06:14:57,742 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:14:57,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741867_1051 (size=13583) 2024-12-15T06:14:57,743 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:14:57,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:14:57,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:14:57,743 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:14:57,743 INFO [RS:1;00a5f8d66132:42161 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:14:57,743 INFO [RS:1;00a5f8d66132:42161 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:14:57,743 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,42161,1734243251265 2024-12-15T06:14:57,743 DEBUG [RS:1;00a5f8d66132:42161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:57,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:57,743 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,42161,1734243251265; all regions closed. 2024-12-15T06:14:57,743 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:14:57,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:57,744 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,42161,1734243251265 2024-12-15T06:14:57,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:14:57,744 WARN [WAL-Shutdown-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=25 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/.tmp/info/0c3fab9904814427be943f6cb1b42dab 2024-12-15T06:14:57,744 ERROR [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1664): Shutdown / close of WAL failed: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... 2024-12-15T06:14:57,744 DEBUG [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1665): Shutdown / close exception details: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,745 DEBUG [RS:1;00a5f8d66132:42161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:57,745 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:14:57,745 INFO [RS:1;00a5f8d66132:42161 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-15T06:14:57,745 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:14:57,745 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:14:57,745 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:14:57,745 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:14:57,745 INFO [RS:1;00a5f8d66132:42161 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42161 2024-12-15T06:14:57,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,42161,1734243251265 2024-12-15T06:14:57,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:14:57,751 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,42161,1734243251265] 2024-12-15T06:14:57,751 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,42161,1734243251265; numProcessing=1 2024-12-15T06:14:57,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/.tmp/info/0c3fab9904814427be943f6cb1b42dab as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/info/0c3fab9904814427be943f6cb1b42dab 2024-12-15T06:14:57,753 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,42161,1734243251265 already deleted, retry=false 2024-12-15T06:14:57,753 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,42161,1734243251265 expired; onlineServers=1 2024-12-15T06:14:57,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/info/0c3fab9904814427be943f6cb1b42dab, entries=8, sequenceid=25, filesize=13.3 K 2024-12-15T06:14:57,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~10.50 KB/10757, heapSize ~11.48 KB/11760, currentSize=9.46 KB/9684 for 69ff2480557a57f1cd7d9a62de1d020c in 33ms, sequenceid=25, compaction requested=false 2024-12-15T06:14:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 69ff2480557a57f1cd7d9a62de1d020c: 2024-12-15T06:14:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=23.4 K, sizeToCheck=16.0 K 2024-12-15T06:14:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:14:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/69ff2480557a57f1cd7d9a62de1d020c/info/0c3fab9904814427be943f6cb1b42dab because midkey is the same as first or last row 2024-12-15T06:14:57,759 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:14:57,759 INFO [RS:0;00a5f8d66132:37129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(3579): Received CLOSE for 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(3579): Received CLOSE for 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,37129,1734243249809 2024-12-15T06:14:57,760 DEBUG [RS:0;00a5f8d66132:37129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:14:57,760 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6f9649a9ab340c5b6adc78efa277393a, disabling compactions & flushes 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:14:57,760 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:57,760 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:57,760 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. after waiting 0 ms 2024-12-15T06:14:57,760 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:57,760 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-15T06:14:57,760 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6f9649a9ab340c5b6adc78efa277393a 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-15T06:14:57,760 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1603): Online Regions={6f9649a9ab340c5b6adc78efa277393a=hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a., 1588230740=hbase:meta,,1.1588230740, 69ff2480557a57f1cd7d9a62de1d020c=TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c.} 2024-12-15T06:14:57,761 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:14:57,761 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 69ff2480557a57f1cd7d9a62de1d020c, 6f9649a9ab340c5b6adc78efa277393a 2024-12-15T06:14:57,761 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:14:57,761 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:14:57,761 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:14:57,761 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:14:57,761 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.87 KB heapSize=5.40 KB 2024-12-15T06:14:57,761 WARN [RS_OPEN_META-regionserver/00a5f8d66132:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,761 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C37129%2C1734243249809.meta:.meta(num 1734243250640) roll requested 2024-12-15T06:14:57,762 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:14:57,762 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37129%2C1734243249809.meta.1734243297762.meta 2024-12-15T06:14:57,762 ERROR [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server 00a5f8d66132,37129,1734243249809: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,762 ERROR [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-15T06:14:57,765 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-15T06:14:57,766 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-15T06:14:57,766 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-15T06:14:57,766 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-15T06:14:57,767 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 273269120 }, "NonHeapMemoryUsage": { "committed": 162856960, "init": 7667712, "max": -1, "used": 161016856 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-15T06:14:57,768 WARN [Thread-813 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1052 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45871 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:60494 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741868_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8]'}, localName='127.0.0.1:39877', datanodeUuid='2d779f54-bfb1-4997-b5d7-1b1579493c1a', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741868_1052 to mirror 127.0.0.1:45871 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:57,768 WARN [Thread-813 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741868_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK], DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:57,768 WARN [Thread-813 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741868_1052 2024-12-15T06:14:57,768 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:60494 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741868_1052] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-15T06:14:57,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:60494 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741868_1052] {}] datanode.DataXceiver(331): 127.0.0.1:39877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60494 dst: /127.0.0.1:39877 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:57,769 WARN [Thread-813 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:57,770 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43079 {}] master.MasterRpcServices(626): 00a5f8d66132,37129,1734243249809 reported a fatal error: ***** ABORTING region server 00a5f8d66132,37129,1734243249809: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** Cause: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) 2024-12-15T06:14:57,770 WARN [Thread-813 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,770 WARN [Thread-813 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741869_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK], DatanodeInfoWithStorage[127.0.0.1:38783,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:57,770 WARN [Thread-813 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741869_1053 2024-12-15T06:14:57,771 WARN [Thread-813 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:57,777 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL 2024-12-15T06:14:57,777 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta with entries=11, filesize=3.63 KB; new WAL /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243297762.meta 2024-12-15T06:14:57,777 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45847:45847),(127.0.0.1/127.0.0.1:35225:35225)] 2024-12-15T06:14:57,777 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta is not closed yet, will try archiving it next time 2024-12-15T06:14:57,777 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,777 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42215,DS-ba6c00cf-e397-4267-b6bc-85bc4bb91c24,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,777 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta 2024-12-15T06:14:57,778 WARN [IPC Server handler 0 on default port 35953 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta has not been closed. Lease recovery is in progress. RecoveryId = 1055 for block blk_1073741834_1010 2024-12-15T06:14:57,778 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta after 1ms 2024-12-15T06:14:57,781 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/.tmp/info/d88ca11c0564487fabf389071a95f35e is 45, key is default/info:d/1734243251107/Put/seqid=0 2024-12-15T06:14:57,782 WARN [Thread-814 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,783 WARN [Thread-814 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741871_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK], DatanodeInfoWithStorage[127.0.0.1:39877,DS-936d601c-7b75-45b8-9970-f1f928b53f73,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK]) is bad. 2024-12-15T06:14:57,783 WARN [Thread-814 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741871_1056 2024-12-15T06:14:57,783 WARN [Thread-814 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45871,DS-b03c15e6-e17c-4148-a059-19d67234119a,DISK] 2024-12-15T06:14:57,785 WARN [Thread-814 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1057 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37061 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:14:57,785 WARN [Thread-814 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1324047213-172.17.0.2-1734243249069:blk_1073741872_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38783,DS-ae2277ff-2795-4fb0-bebf-73e886f57a05,DISK], DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK]) is bad. 2024-12-15T06:14:57,785 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:59142 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741872_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data4]'}, localName='127.0.0.1:38783', datanodeUuid='8951ac71-220e-4e09-bd29-42ad8229db01', xmitsInProgress=0}:Exception transferring block BP-1324047213-172.17.0.2-1734243249069:blk_1073741872_1057 to mirror 127.0.0.1:37061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:57,785 WARN [Thread-814 {}] hdfs.DataStreamer(1850): Abandoning BP-1324047213-172.17.0.2-1734243249069:blk_1073741872_1057 2024-12-15T06:14:57,785 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:59142 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741872_1057] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-15T06:14:57,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-887837040_22 at /127.0.0.1:59142 [Receiving block BP-1324047213-172.17.0.2-1734243249069:blk_1073741872_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38783:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59142 dst: /127.0.0.1:38783 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:14:57,786 WARN [Thread-814 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37061,DS-dc2e1232-6a91-4577-83f4-34269edc2fd2,DISK] 2024-12-15T06:14:57,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741873_1058 (size=5037) 2024-12-15T06:14:57,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741873_1058 (size=5037) 2024-12-15T06:14:57,790 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/.tmp/info/d88ca11c0564487fabf389071a95f35e 2024-12-15T06:14:57,797 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/.tmp/info/d88ca11c0564487fabf389071a95f35e as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/info/d88ca11c0564487fabf389071a95f35e 2024-12-15T06:14:57,802 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/info/d88ca11c0564487fabf389071a95f35e, entries=2, sequenceid=6, filesize=4.9 K 2024-12-15T06:14:57,803 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 6f9649a9ab340c5b6adc78efa277393a in 43ms, sequenceid=6, compaction requested=false 2024-12-15T06:14:57,807 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/data/hbase/namespace/6f9649a9ab340c5b6adc78efa277393a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-15T06:14:57,808 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6f9649a9ab340c5b6adc78efa277393a: 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734243250692.6f9649a9ab340c5b6adc78efa277393a. 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 69ff2480557a57f1cd7d9a62de1d020c, disabling compactions & flushes 2024-12-15T06:14:57,808 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. after waiting 0 ms 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 69ff2480557a57f1cd7d9a62de1d020c: 2024-12-15T06:14:57,808 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:14:57,849 INFO [RS:1;00a5f8d66132:42161 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,42161,1734243251265; zookeeper connection closed. 2024-12-15T06:14:57,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42161-0x100944fa7300003, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:14:57,849 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67b2156a {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67b2156a 2024-12-15T06:14:57,961 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:14:57,961 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(3579): Received CLOSE for 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:57,961 DEBUG [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 69ff2480557a57f1cd7d9a62de1d020c 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 69ff2480557a57f1cd7d9a62de1d020c, disabling compactions & flushes 2024-12-15T06:14:57,961 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:14:57,961 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. after waiting 0 ms 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 69ff2480557a57f1cd7d9a62de1d020c: 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing TestLogRolling-testLogRollOnDatanodeDeath,,1734243251364.69ff2480557a57f1cd7d9a62de1d020c. 2024-12-15T06:14:57,961 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740 2024-12-15T06:14:58,095 INFO [regionserver/00a5f8d66132:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:14:58,124 INFO [regionserver/00a5f8d66132:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-15T06:14:58,124 INFO [regionserver/00a5f8d66132:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-15T06:14:58,126 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243250239 to hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs/00a5f8d66132%2C37129%2C1734243249809.1734243250239 2024-12-15T06:14:58,127 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243274072 to hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs/00a5f8d66132%2C37129%2C1734243249809.1734243274072 2024-12-15T06:14:58,128 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278084 to hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs/00a5f8d66132%2C37129%2C1734243249809.1734243278084 2024-12-15T06:14:58,129 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.1734243278303 to hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/oldWALs/00a5f8d66132%2C37129%2C1734243249809.1734243278303 2024-12-15T06:14:58,161 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing 2024-12-15T06:14:58,161 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,37129,1734243249809; all regions closed. 2024-12-15T06:14:58,162 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809 2024-12-15T06:14:58,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741870_1054 (size=93) 2024-12-15T06:14:58,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741870_1054 (size=93) 2024-12-15T06:14:58,959 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2b42e84b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1324047213-172.17.0.2-1734243249069:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:42215,null,null]) java.net.ConnectException: Call From 00a5f8d66132/172.17.0.2 to localhost:34335 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-15T06:14:59,328 INFO [regionserver/00a5f8d66132:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:15:00,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741860_1043 (size=1618) 2024-12-15T06:15:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741830_1048 (size=47136) 2024-12-15T06:15:01,779 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta after 4002ms 2024-12-15T06:15:02,809 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:02,820 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:02,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:02,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:02,830 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:02,832 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,166 ERROR [WAL-Shutdown-0 {}] wal.FSHLog(508): We have waited 5 seconds but the close of writer(s) doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-15T06:15:03,167 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809 2024-12-15T06:15:03,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741865_1049 (size=13280) 2024-12-15T06:15:03,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741865_1049 (size=13280) 2024-12-15T06:15:03,170 DEBUG [RS:0;00a5f8d66132:37129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:03,170 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:15:03,171 INFO [RS:0;00a5f8d66132:37129 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-15T06:15:03,171 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:15:03,171 INFO [RS:0;00a5f8d66132:37129 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:37129 2024-12-15T06:15:03,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,37129,1734243249809 2024-12-15T06:15:03,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:15:03,174 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,37129,1734243249809] 2024-12-15T06:15:03,174 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,37129,1734243249809; numProcessing=2 2024-12-15T06:15:03,176 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,37129,1734243249809 already deleted, retry=false 2024-12-15T06:15:03,176 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,37129,1734243249809 expired; onlineServers=0 2024-12-15T06:15:03,176 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,43079,1734243249755' ***** 2024-12-15T06:15:03,176 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-15T06:15:03,176 DEBUG [M:0;00a5f8d66132:43079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c41aace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:15:03,176 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,43079,1734243249755 2024-12-15T06:15:03,176 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,43079,1734243249755; all regions closed. 2024-12-15T06:15:03,176 DEBUG [M:0;00a5f8d66132:43079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:03,176 DEBUG [M:0;00a5f8d66132:43079 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-15T06:15:03,176 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-15T06:15:03,176 DEBUG [M:0;00a5f8d66132:43079 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-15T06:15:03,176 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243250007 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243250007,5,FailOnTimeoutGroup] 2024-12-15T06:15:03,176 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243250007 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243250007,5,FailOnTimeoutGroup] 2024-12-15T06:15:03,177 INFO [M:0;00a5f8d66132:43079 {}] hbase.ChoreService(370): Chore service for: master/00a5f8d66132:0 had [] on shutdown 2024-12-15T06:15:03,177 DEBUG [M:0;00a5f8d66132:43079 {}] master.HMaster(1733): Stopping service threads 2024-12-15T06:15:03,177 INFO [M:0;00a5f8d66132:43079 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-15T06:15:03,177 INFO [M:0;00a5f8d66132:43079 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-15T06:15:03,177 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-15T06:15:03,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-15T06:15:03,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:03,178 DEBUG [M:0;00a5f8d66132:43079 {}] zookeeper.ZKUtil(347): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-15T06:15:03,178 WARN [M:0;00a5f8d66132:43079 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-15T06:15:03,178 INFO [M:0;00a5f8d66132:43079 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-15T06:15:03,178 INFO [M:0;00a5f8d66132:43079 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-15T06:15:03,178 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:15:03,178 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:03,178 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:03,178 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:15:03,178 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:03,178 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.07 KB heapSize=49.27 KB 2024-12-15T06:15:03,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:15:03,198 DEBUG [M:0;00a5f8d66132:43079 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ff32772fd4449e48f5be2da59773d5b is 82, key is hbase:meta,,1/info:regioninfo/1734243250667/Put/seqid=0 2024-12-15T06:15:03,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741874_1059 (size=5672) 2024-12-15T06:15:03,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741874_1059 (size=5672) 2024-12-15T06:15:03,204 INFO [M:0;00a5f8d66132:43079 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ff32772fd4449e48f5be2da59773d5b 2024-12-15T06:15:03,224 DEBUG [M:0;00a5f8d66132:43079 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19a7af5bcfee416386d0710a75420095 is 773, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734243251746/Put/seqid=0 2024-12-15T06:15:03,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741875_1060 (size=7464) 2024-12-15T06:15:03,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741875_1060 (size=7464) 2024-12-15T06:15:03,229 INFO [M:0;00a5f8d66132:43079 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.40 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19a7af5bcfee416386d0710a75420095 2024-12-15T06:15:03,249 DEBUG [M:0;00a5f8d66132:43079 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7bc42bf53b454b11b7ef9115ff903f7b is 69, key is 00a5f8d66132,37129,1734243249809/rs:state/1734243250074/Put/seqid=0 2024-12-15T06:15:03,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741876_1061 (size=5224) 2024-12-15T06:15:03,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741876_1061 (size=5224) 2024-12-15T06:15:03,254 INFO [M:0;00a5f8d66132:43079 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7bc42bf53b454b11b7ef9115ff903f7b 2024-12-15T06:15:03,273 DEBUG [M:0;00a5f8d66132:43079 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f22d2ff001a24c90a8eb1b3511ad5aba is 52, key is load_balancer_on/state:d/1734243251247/Put/seqid=0 2024-12-15T06:15:03,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:03,274 INFO [RS:0;00a5f8d66132:37129 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,37129,1734243249809; zookeeper connection closed. 2024-12-15T06:15:03,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x100944fa7300001, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:03,275 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31ab4954 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31ab4954 2024-12-15T06:15:03,275 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-15T06:15:03,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741877_1062 (size=5056) 2024-12-15T06:15:03,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741877_1062 (size=5056) 2024-12-15T06:15:03,279 INFO [M:0;00a5f8d66132:43079 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f22d2ff001a24c90a8eb1b3511ad5aba 2024-12-15T06:15:03,284 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ff32772fd4449e48f5be2da59773d5b as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4ff32772fd4449e48f5be2da59773d5b 2024-12-15T06:15:03,290 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4ff32772fd4449e48f5be2da59773d5b, entries=8, sequenceid=97, filesize=5.5 K 2024-12-15T06:15:03,291 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19a7af5bcfee416386d0710a75420095 as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/19a7af5bcfee416386d0710a75420095 2024-12-15T06:15:03,296 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/19a7af5bcfee416386d0710a75420095, entries=11, sequenceid=97, filesize=7.3 K 2024-12-15T06:15:03,297 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7bc42bf53b454b11b7ef9115ff903f7b as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7bc42bf53b454b11b7ef9115ff903f7b 2024-12-15T06:15:03,302 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7bc42bf53b454b11b7ef9115ff903f7b, entries=2, sequenceid=97, filesize=5.1 K 2024-12-15T06:15:03,303 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f22d2ff001a24c90a8eb1b3511ad5aba as hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f22d2ff001a24c90a8eb1b3511ad5aba 2024-12-15T06:15:03,307 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f22d2ff001a24c90a8eb1b3511ad5aba, entries=1, sequenceid=97, filesize=4.9 K 2024-12-15T06:15:03,308 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.07 KB/41027, heapSize ~49.20 KB/50384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 130ms, sequenceid=97, compaction requested=false 2024-12-15T06:15:03,310 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:03,310 DEBUG [M:0;00a5f8d66132:43079 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:15:03,310 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/MasterData/WALs/00a5f8d66132,43079,1734243249755 2024-12-15T06:15:03,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39877 is added to blk_1073741864_1047 (size=757) 2024-12-15T06:15:03,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38783 is added to blk_1073741864_1047 (size=757) 2024-12-15T06:15:03,313 INFO [M:0;00a5f8d66132:43079 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-15T06:15:03,313 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:15:03,313 INFO [M:0;00a5f8d66132:43079 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43079 2024-12-15T06:15:03,314 DEBUG [M:0;00a5f8d66132:43079 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/00a5f8d66132,43079,1734243249755 already deleted, retry=false 2024-12-15T06:15:03,334 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:15:03,335 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,350 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,351 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,354 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,354 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,354 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:03,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:03,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43079-0x100944fa7300000, quorum=127.0.0.1:51726, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:03,416 INFO [M:0;00a5f8d66132:43079 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,43079,1734243249755; zookeeper connection closed. 2024-12-15T06:15:03,420 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4f1d6d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:03,420 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43126ed{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:03,420 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:03,421 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5697186b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:03,421 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34c7ff9b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:03,422 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@a49809b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1324047213-172.17.0.2-1734243249069:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:42215,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:34335 , LocalHost:localPort 00a5f8d66132/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-15T06:15:03,423 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1324047213-172.17.0.2-1734243249069 (Datanode Uuid 8951ac71-220e-4e09-bd29-42ad8229db01) service to localhost/127.0.0.1:35953 2024-12-15T06:15:03,424 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data3/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:03,424 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data4/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:03,425 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:03,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30f14291{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:03,427 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@19dd2382{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:03,427 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:03,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1051bc4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:03,428 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dd74f13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:03,429 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:03,429 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:03,429 WARN [BP-1324047213-172.17.0.2-1734243249069 heartbeating to localhost/127.0.0.1:35953 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1324047213-172.17.0.2-1734243249069 (Datanode Uuid 2d779f54-bfb1-4997-b5d7-1b1579493c1a) service to localhost/127.0.0.1:35953 2024-12-15T06:15:03,429 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:03,430 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data7/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:03,430 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/cluster_74912040-c728-fd45-f96e-0c41331d35ed/dfs/data/data8/current/BP-1324047213-172.17.0.2-1734243249069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:03,430 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:03,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9614fc0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:15:03,438 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4403aa3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:03,438 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:03,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c5b422f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:03,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a305303{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:03,449 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-15T06:15:03,481 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-15T06:15:03,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=87 (was 66) Potentially hanging thread: nioEventLoopGroup-13-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35953 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35953 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:35953 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$$Lambda$793/0x00007f7838b66db0.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35953 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:35953 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Command processor java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411) app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RS-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:35953 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:35953 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:35953 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35953 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=428 (was 403) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=123 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=4435 (was 4835) 2024-12-15T06:15:03,495 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=87, OpenFileDescriptor=428, MaxFileDescriptor=1048576, SystemLoadAverage=123, ProcessCount=11, AvailableMemoryMB=4435 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.log.dir so I do NOT create it in target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b28270a2-4234-fc98-4db5-4ecd691532e8/hadoop.tmp.dir so I do NOT create it in target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e, deleteOnExit=true 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/test.cache.data in system properties and HBase conf 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.tmp.dir in system properties and HBase conf 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir in system properties and HBase conf 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-15T06:15:03,496 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-15T06:15:03,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:15:03,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-15T06:15:03,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/nfs.dump.dir in system properties and HBase conf 2024-12-15T06:15:03,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir in system properties and HBase conf 2024-12-15T06:15:03,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:15:03,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-15T06:15:03,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-15T06:15:03,511 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:15:03,579 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:03,583 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:03,584 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:03,585 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:03,585 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:15:03,585 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:03,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16bb361{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:03,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53b34952{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:03,703 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d8de0cb{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-40405-hadoop-hdfs-3_4_1-tests_jar-_-any-16249579840980601517/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:15:03,704 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@723e44b3{HTTP/1.1, (http/1.1)}{localhost:40405} 2024-12-15T06:15:03,704 INFO [Time-limited test {}] server.Server(415): Started @180833ms 2024-12-15T06:15:03,718 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:15:03,780 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:03,780 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:03,784 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:03,785 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:03,785 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:03,785 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:15:03,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@447aae3f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:03,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34816a02{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:03,901 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b9c2d96{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-35997-hadoop-hdfs-3_4_1-tests_jar-_-any-183226295117949509/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:03,901 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cf9d8d5{HTTP/1.1, (http/1.1)}{localhost:35997} 2024-12-15T06:15:03,901 INFO [Time-limited test {}] server.Server(415): Started @181030ms 2024-12-15T06:15:03,902 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:03,934 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:03,937 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:03,937 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:03,937 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:03,938 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:15:03,938 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a89dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:03,939 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@673c93a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:03,984 WARN [Thread-933 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data1/current/BP-164817931-172.17.0.2-1734243303529/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:03,984 WARN [Thread-934 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data2/current/BP-164817931-172.17.0.2-1734243303529/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:04,005 WARN [Thread-912 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2219d484ebfebde5 with lease ID 0x9d65686a2bfe348d: Processing first storage report for DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034 from datanode DatanodeRegistration(127.0.0.1:42323, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=46509, infoSecurePort=0, ipcPort=34235, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529) 2024-12-15T06:15:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2219d484ebfebde5 with lease ID 0x9d65686a2bfe348d: from storage DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034 node DatanodeRegistration(127.0.0.1:42323, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=46509, infoSecurePort=0, ipcPort=34235, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2219d484ebfebde5 with lease ID 0x9d65686a2bfe348d: Processing first storage report for DS-1b6e478d-72cd-4e00-b498-6ca55215448b from datanode DatanodeRegistration(127.0.0.1:42323, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=46509, infoSecurePort=0, ipcPort=34235, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529) 2024-12-15T06:15:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2219d484ebfebde5 with lease ID 0x9d65686a2bfe348d: from storage DS-1b6e478d-72cd-4e00-b498-6ca55215448b node DatanodeRegistration(127.0.0.1:42323, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=46509, infoSecurePort=0, ipcPort=34235, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:04,056 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e00649{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-39329-hadoop-hdfs-3_4_1-tests_jar-_-any-14530566832453585298/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:04,057 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6877c523{HTTP/1.1, (http/1.1)}{localhost:39329} 2024-12-15T06:15:04,057 INFO [Time-limited test {}] server.Server(415): Started @181186ms 2024-12-15T06:15:04,058 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:04,148 WARN [Thread-959 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data3/current/BP-164817931-172.17.0.2-1734243303529/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:04,148 WARN [Thread-960 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data4/current/BP-164817931-172.17.0.2-1734243303529/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:04,166 WARN [Thread-948 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3032818b9c2e0f51 with lease ID 0x9d65686a2bfe348e: Processing first storage report for DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d from datanode DatanodeRegistration(127.0.0.1:33603, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=45925, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529) 2024-12-15T06:15:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3032818b9c2e0f51 with lease ID 0x9d65686a2bfe348e: from storage DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d node DatanodeRegistration(127.0.0.1:33603, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=45925, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3032818b9c2e0f51 with lease ID 0x9d65686a2bfe348e: Processing first storage report for DS-be4f8080-1ca9-4a8c-95c6-73f0f355e300 from datanode DatanodeRegistration(127.0.0.1:33603, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=45925, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529) 2024-12-15T06:15:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3032818b9c2e0f51 with lease ID 0x9d65686a2bfe348e: from storage DS-be4f8080-1ca9-4a8c-95c6-73f0f355e300 node DatanodeRegistration(127.0.0.1:33603, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=45925, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:04,185 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b 2024-12-15T06:15:04,187 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/zookeeper_0, clientPort=60839, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-15T06:15:04,188 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60839 2024-12-15T06:15:04,188 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:15:04,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:15:04,200 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e with version=8 2024-12-15T06:15:04,200 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase-staging 2024-12-15T06:15:04,202 INFO [Time-limited test {}] client.ConnectionUtils(129): master/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:15:04,202 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:15:04,203 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36391 2024-12-15T06:15:04,203 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,205 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,207 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36391 connecting to ZooKeeper ensemble=127.0.0.1:60839 2024-12-15T06:15:04,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:363910x0, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:15:04,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36391-0x10094507be10000 connected 2024-12-15T06:15:04,226 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:15:04,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:15:04,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:15:04,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36391 2024-12-15T06:15:04,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36391 2024-12-15T06:15:04,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36391 2024-12-15T06:15:04,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36391 2024-12-15T06:15:04,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36391 2024-12-15T06:15:04,233 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e, hbase.cluster.distributed=false 2024-12-15T06:15:04,250 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:15:04,250 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:15:04,251 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41173 2024-12-15T06:15:04,251 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:15:04,252 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:15:04,252 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,254 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,257 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:41173 connecting to ZooKeeper ensemble=127.0.0.1:60839 2024-12-15T06:15:04,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411730x0, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:15:04,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41173-0x10094507be10001 connected 2024-12-15T06:15:04,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:15:04,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:15:04,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:15:04,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41173 2024-12-15T06:15:04,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41173 2024-12-15T06:15:04,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41173 2024-12-15T06:15:04,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41173 2024-12-15T06:15:04,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41173 2024-12-15T06:15:04,266 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:04,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:04,268 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:15:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:15:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,270 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:15:04,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:15:04,272 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00a5f8d66132,36391,1734243304201 from backup master directory 2024-12-15T06:15:04,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:04,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:04,273 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:15:04,273 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,278 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00a5f8d66132:36391 2024-12-15T06:15:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:15:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:15:04,285 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/hbase.id with ID: 5c019415-f370-4135-a31e-046e3655b9dd 2024-12-15T06:15:04,296 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:04,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:15:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:15:04,307 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:15:04,307 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-15T06:15:04,308 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:15:04,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:15:04,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:15:04,316 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store 2024-12-15T06:15:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:15:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:15:04,322 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:04,322 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:15:04,322 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:04,322 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:04,322 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:15:04,322 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:04,322 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:04,323 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:15:04,323 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/.initializing 2024-12-15T06:15:04,323 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,326 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C36391%2C1734243304201, suffix=, logDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/oldWALs, maxLogs=10 2024-12-15T06:15:04,327 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36391%2C1734243304201.1734243304326 2024-12-15T06:15:04,331 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 2024-12-15T06:15:04,332 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46509:46509),(127.0.0.1/127.0.0.1:45925:45925)] 2024-12-15T06:15:04,332 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:04,332 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:04,332 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,332 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,333 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-15T06:15:04,335 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:04,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-15T06:15:04,336 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:04,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-15T06:15:04,338 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:04,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-15T06:15:04,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:04,340 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,341 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,342 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-15T06:15:04,343 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:04,345 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:04,346 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=833524, jitterRate=0.0598815381526947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-15T06:15:04,346 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:15:04,347 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-15T06:15:04,350 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5007e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:15:04,350 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-15T06:15:04,351 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-15T06:15:04,351 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-15T06:15:04,351 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-15T06:15:04,351 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-15T06:15:04,351 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-15T06:15:04,351 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-15T06:15:04,353 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-15T06:15:04,354 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-15T06:15:04,356 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-15T06:15:04,356 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-15T06:15:04,357 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-15T06:15:04,359 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-15T06:15:04,360 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-15T06:15:04,360 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-15T06:15:04,363 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-15T06:15:04,364 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-15T06:15:04,366 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-15T06:15:04,367 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-15T06:15:04,369 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-15T06:15:04,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:15:04,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:15:04,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=00a5f8d66132,36391,1734243304201, sessionid=0x10094507be10000, setting cluster-up flag (Was=false) 2024-12-15T06:15:04,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,382 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-15T06:15:04,383 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,393 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-15T06:15:04,394 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,36391,1734243304201 2024-12-15T06:15:04,396 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-15T06:15:04,397 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-15T06:15:04,397 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00a5f8d66132,36391,1734243304201 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00a5f8d66132:0, corePoolSize=10, maxPoolSize=10 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:15:04,397 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,399 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:15:04,399 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734243334400 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-15T06:15:04,400 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-15T06:15:04,400 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:15:04,400 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,401 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-15T06:15:04,401 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-15T06:15:04,401 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-15T06:15:04,402 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-15T06:15:04,402 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-15T06:15:04,402 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243304402,5,FailOnTimeoutGroup] 2024-12-15T06:15:04,402 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243304402,5,FailOnTimeoutGroup] 2024-12-15T06:15:04,402 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,402 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-15T06:15:04,402 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,402 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:15:04,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:15:04,411 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-15T06:15:04,411 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e 2024-12-15T06:15:04,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:15:04,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:15:04,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:04,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:15:04,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:15:04,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:04,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:15:04,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:15:04,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:04,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:15:04,424 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:15:04,424 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:04,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:04,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/meta/1588230740 2024-12-15T06:15:04,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/meta/1588230740 2024-12-15T06:15:04,427 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:15:04,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:15:04,430 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:04,430 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796171, jitterRate=0.012384876608848572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:15:04,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:15:04,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:15:04,431 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:15:04,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:15:04,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:15:04,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:15:04,431 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:15:04,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:15:04,432 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:15:04,433 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-15T06:15:04,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-15T06:15:04,434 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-15T06:15:04,435 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-15T06:15:04,478 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00a5f8d66132:41173 2024-12-15T06:15:04,479 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1008): ClusterId : 5c019415-f370-4135-a31e-046e3655b9dd 2024-12-15T06:15:04,479 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:15:04,482 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:15:04,482 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:15:04,483 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:15:04,484 DEBUG [RS:0;00a5f8d66132:41173 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc39efd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:15:04,484 DEBUG [RS:0;00a5f8d66132:41173 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b7939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:15:04,485 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:15:04,485 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:15:04,485 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:15:04,485 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,36391,1734243304201 with isa=00a5f8d66132/172.17.0.2:41173, startcode=1734243304249 2024-12-15T06:15:04,485 DEBUG [RS:0;00a5f8d66132:41173 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:15:04,487 INFO [RS-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52153, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:15:04,488 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36391 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,488 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36391 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,489 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e 2024-12-15T06:15:04,489 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:41731 2024-12-15T06:15:04,489 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:15:04,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:15:04,491 DEBUG [RS:0;00a5f8d66132:41173 {}] zookeeper.ZKUtil(111): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,491 WARN [RS:0;00a5f8d66132:41173 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:15:04,492 INFO [RS:0;00a5f8d66132:41173 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:15:04,492 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,492 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,41173,1734243304249] 2024-12-15T06:15:04,495 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:15:04,495 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:15:04,496 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:15:04,497 INFO [RS:0;00a5f8d66132:41173 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:15:04,497 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,497 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:15:04,498 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:15:04,498 DEBUG [RS:0;00a5f8d66132:41173 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:15:04,500 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,500 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,500 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,500 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,500 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,41173,1734243304249-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:15:04,515 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:15:04,515 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,41173,1734243304249-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:04,530 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.Replication(204): 00a5f8d66132,41173,1734243304249 started 2024-12-15T06:15:04,530 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,41173,1734243304249, RpcServer on 00a5f8d66132/172.17.0.2:41173, sessionid=0x10094507be10001 2024-12-15T06:15:04,530 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:15:04,530 DEBUG [RS:0;00a5f8d66132:41173 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,530 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,41173,1734243304249' 2024-12-15T06:15:04,530 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:15:04,530 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,41173,1734243304249' 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:15:04,531 DEBUG [RS:0;00a5f8d66132:41173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:15:04,531 INFO [RS:0;00a5f8d66132:41173 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:15:04,531 INFO [RS:0;00a5f8d66132:41173 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:15:04,585 WARN [00a5f8d66132:36391 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-15T06:15:04,633 INFO [RS:0;00a5f8d66132:41173 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C41173%2C1734243304249, suffix=, logDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/oldWALs, maxLogs=32 2024-12-15T06:15:04,635 INFO [RS:0;00a5f8d66132:41173 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:04,640 INFO [RS:0;00a5f8d66132:41173 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:04,640 DEBUG [RS:0;00a5f8d66132:41173 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45925:45925),(127.0.0.1/127.0.0.1:46509:46509)] 2024-12-15T06:15:04,781 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:04,835 DEBUG [00a5f8d66132:36391 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-15T06:15:04,836 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,837 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,41173,1734243304249, state=OPENING 2024-12-15T06:15:04,839 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-15T06:15:04,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:04,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=00a5f8d66132,41173,1734243304249}] 2024-12-15T06:15:04,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:04,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:04,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-15T06:15:04,994 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:04,994 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-15T06:15:04,996 INFO [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55362, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-15T06:15:04,999 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-15T06:15:04,999 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:15:05,001 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C41173%2C1734243304249.meta, suffix=.meta, logDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/oldWALs, maxLogs=32 2024-12-15T06:15:05,002 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta 2024-12-15T06:15:05,007 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta 2024-12-15T06:15:05,007 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45925:45925),(127.0.0.1/127.0.0.1:46509:46509)] 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-15T06:15:05,008 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-15T06:15:05,008 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-15T06:15:05,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:15:05,010 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:15:05,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:05,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:15:05,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:15:05,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:05,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:15:05,013 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:15:05,013 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:05,014 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/meta/1588230740 2024-12-15T06:15:05,015 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/meta/1588230740 2024-12-15T06:15:05,017 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:15:05,018 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:15:05,019 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881464, jitterRate=0.12084010243415833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:15:05,019 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:15:05,020 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734243304993 2024-12-15T06:15:05,022 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-15T06:15:05,022 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-15T06:15:05,023 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,023 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,41173,1734243304249, state=OPEN 2024-12-15T06:15:05,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:15:05,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:15:05,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:05,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:05,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-15T06:15:05,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=00a5f8d66132,41173,1734243304249 in 186 msec 2024-12-15T06:15:05,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-15T06:15:05,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 596 msec 2024-12-15T06:15:05,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 637 msec 2024-12-15T06:15:05,033 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734243305033, completionTime=-1 2024-12-15T06:15:05,034 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-15T06:15:05,034 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-15T06:15:05,034 DEBUG [hconnection-0x667b0f1f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:15:05,035 INFO [RS-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55368, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:15:05,036 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-15T06:15:05,036 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734243365036 2024-12-15T06:15:05,036 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734243425036 2024-12-15T06:15:05,037 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36391,1734243304201-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36391,1734243304201-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36391,1734243304201-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00a5f8d66132:36391, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-15T06:15:05,042 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:15:05,043 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-15T06:15:05,043 DEBUG [master/00a5f8d66132:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-15T06:15:05,044 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:15:05,045 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,045 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:15:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:15:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:15:05,055 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 036d6e4c75825a641cbf43445e948636, NAME => 'hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e 2024-12-15T06:15:05,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:15:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:15:05,062 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:05,062 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 036d6e4c75825a641cbf43445e948636, disabling compactions & flushes 2024-12-15T06:15:05,062 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,062 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,062 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. after waiting 0 ms 2024-12-15T06:15:05,062 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,062 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,062 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 036d6e4c75825a641cbf43445e948636: 2024-12-15T06:15:05,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:15:05,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734243305064"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243305064"}]},"ts":"1734243305064"} 2024-12-15T06:15:05,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:15:05,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:15:05,067 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243305067"}]},"ts":"1734243305067"} 2024-12-15T06:15:05,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-15T06:15:05,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=036d6e4c75825a641cbf43445e948636, ASSIGN}] 2024-12-15T06:15:05,073 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=036d6e4c75825a641cbf43445e948636, ASSIGN 2024-12-15T06:15:05,074 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=036d6e4c75825a641cbf43445e948636, ASSIGN; state=OFFLINE, location=00a5f8d66132,41173,1734243304249; forceNewPlan=false, retain=false 2024-12-15T06:15:05,225 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=036d6e4c75825a641cbf43445e948636, regionState=OPENING, regionLocation=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 036d6e4c75825a641cbf43445e948636, server=00a5f8d66132,41173,1734243304249}] 2024-12-15T06:15:05,380 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,384 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,384 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 036d6e4c75825a641cbf43445e948636, NAME => 'hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:05,384 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,384 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:05,385 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,385 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,386 INFO [StoreOpener-036d6e4c75825a641cbf43445e948636-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,387 INFO [StoreOpener-036d6e4c75825a641cbf43445e948636-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 036d6e4c75825a641cbf43445e948636 columnFamilyName info 2024-12-15T06:15:05,387 DEBUG [StoreOpener-036d6e4c75825a641cbf43445e948636-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,388 INFO [StoreOpener-036d6e4c75825a641cbf43445e948636-1 {}] regionserver.HStore(327): Store=036d6e4c75825a641cbf43445e948636/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:05,389 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/namespace/036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,389 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/namespace/036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:05,393 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/hbase/namespace/036d6e4c75825a641cbf43445e948636/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:05,393 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 036d6e4c75825a641cbf43445e948636; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825134, jitterRate=0.04921320080757141}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:15:05,394 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 036d6e4c75825a641cbf43445e948636: 2024-12-15T06:15:05,395 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636., pid=6, masterSystemTime=1734243305380 2024-12-15T06:15:05,397 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,397 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:05,397 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=036d6e4c75825a641cbf43445e948636, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-15T06:15:05,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 036d6e4c75825a641cbf43445e948636, server=00a5f8d66132,41173,1734243304249 in 172 msec 2024-12-15T06:15:05,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-15T06:15:05,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=036d6e4c75825a641cbf43445e948636, ASSIGN in 329 msec 2024-12-15T06:15:05,405 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:15:05,405 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243305405"}]},"ts":"1734243305405"} 2024-12-15T06:15:05,406 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-15T06:15:05,409 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:15:05,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 367 msec 2024-12-15T06:15:05,444 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-15T06:15:05,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:05,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:15:05,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:05,450 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-15T06:15:05,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:15:05,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 11 msec 2024-12-15T06:15:05,472 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-15T06:15:05,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:15:05,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 11 msec 2024-12-15T06:15:05,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-15T06:15:05,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-15T06:15:05,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.226sec 2024-12-15T06:15:05,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-15T06:15:05,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-15T06:15:05,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-15T06:15:05,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-15T06:15:05,499 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-15T06:15:05,500 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36391,1734243304201-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:15:05,500 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36391,1734243304201-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-15T06:15:05,501 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-15T06:15:05,501 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-15T06:15:05,502 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36391,1734243304201-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:05,568 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42e4c0dc to 127.0.0.1:60839 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@470a3d27 2024-12-15T06:15:05,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613943df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:15:05,574 DEBUG [hconnection-0x774cb9da-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:15:05,575 INFO [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55376, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:15:05,577 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=00a5f8d66132,36391,1734243304201 2024-12-15T06:15:05,577 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:05,581 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-15T06:15:05,581 INFO [Time-limited test {}] wal.TestLogRolling(297): Starting testLogRollOnPipelineRestart 2024-12-15T06:15:05,581 INFO [Time-limited test {}] wal.TestLogRolling(300): Replication=2 2024-12-15T06:15:05,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-15T06:15:05,584 INFO [RS-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43858, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-15T06:15:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-15T06:15:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-15T06:15:05,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:15:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-12-15T06:15:05,587 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:15:05,588 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 9 2024-12-15T06:15:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:15:05,588 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:15:05,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741837_1013 (size=395) 2024-12-15T06:15:05,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741837_1013 (size=395) 2024-12-15T06:15:05,598 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5bc9f5103f6f6ae27c9adb5cb68206ad, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e 2024-12-15T06:15:05,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741838_1014 (size=78) 2024-12-15T06:15:05,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42323 is added to blk_1073741838_1014 (size=78) 2024-12-15T06:15:05,610 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:05,610 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1681): Closing 5bc9f5103f6f6ae27c9adb5cb68206ad, disabling compactions & flushes 2024-12-15T06:15:05,610 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,610 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,610 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. after waiting 0 ms 2024-12-15T06:15:05,610 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,610 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,610 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5bc9f5103f6f6ae27c9adb5cb68206ad: 2024-12-15T06:15:05,611 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:15:05,612 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1734243305612"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243305612"}]},"ts":"1734243305612"} 2024-12-15T06:15:05,614 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:15:05,615 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:15:05,615 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243305615"}]},"ts":"1734243305615"} 2024-12-15T06:15:05,616 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-12-15T06:15:05,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5bc9f5103f6f6ae27c9adb5cb68206ad, ASSIGN}] 2024-12-15T06:15:05,622 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5bc9f5103f6f6ae27c9adb5cb68206ad, ASSIGN 2024-12-15T06:15:05,622 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5bc9f5103f6f6ae27c9adb5cb68206ad, ASSIGN; state=OFFLINE, location=00a5f8d66132,41173,1734243304249; forceNewPlan=false, retain=false 2024-12-15T06:15:05,773 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=5bc9f5103f6f6ae27c9adb5cb68206ad, regionState=OPENING, regionLocation=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 5bc9f5103f6f6ae27c9adb5cb68206ad, server=00a5f8d66132,41173,1734243304249}] 2024-12-15T06:15:05,782 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:05,928 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,932 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,932 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 5bc9f5103f6f6ae27c9adb5cb68206ad, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:05,933 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,933 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:05,933 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,933 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,934 INFO [StoreOpener-5bc9f5103f6f6ae27c9adb5cb68206ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,936 INFO [StoreOpener-5bc9f5103f6f6ae27c9adb5cb68206ad-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bc9f5103f6f6ae27c9adb5cb68206ad columnFamilyName info 2024-12-15T06:15:05,936 DEBUG [StoreOpener-5bc9f5103f6f6ae27c9adb5cb68206ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:05,936 INFO [StoreOpener-5bc9f5103f6f6ae27c9adb5cb68206ad-1 {}] regionserver.HStore(327): Store=5bc9f5103f6f6ae27c9adb5cb68206ad/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:05,937 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/default/TestLogRolling-testLogRollOnPipelineRestart/5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,937 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/default/TestLogRolling-testLogRollOnPipelineRestart/5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,939 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:05,941 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/data/default/TestLogRolling-testLogRollOnPipelineRestart/5bc9f5103f6f6ae27c9adb5cb68206ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:05,942 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 5bc9f5103f6f6ae27c9adb5cb68206ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=772286, jitterRate=-0.017988458275794983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:15:05,942 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 5bc9f5103f6f6ae27c9adb5cb68206ad: 2024-12-15T06:15:05,943 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad., pid=11, masterSystemTime=1734243305928 2024-12-15T06:15:05,945 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,945 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:05,946 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=5bc9f5103f6f6ae27c9adb5cb68206ad, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,41173,1734243304249 2024-12-15T06:15:05,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-15T06:15:05,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 5bc9f5103f6f6ae27c9adb5cb68206ad, server=00a5f8d66132,41173,1734243304249 in 172 msec 2024-12-15T06:15:05,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-15T06:15:05,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=5bc9f5103f6f6ae27c9adb5cb68206ad, ASSIGN in 330 msec 2024-12-15T06:15:05,953 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:15:05,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243305953"}]},"ts":"1734243305953"} 2024-12-15T06:15:05,954 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-12-15T06:15:05,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:15:05,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 372 msec 2024-12-15T06:15:06,783 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:07,783 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:08,784 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:09,785 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:10,423 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:15:10,435 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,435 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,437 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:10,495 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-15T06:15:10,496 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-12-15T06:15:10,786 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:11,786 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:12,787 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:13,788 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:14,788 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:14,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-15T06:15:14,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-15T06:15:14,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-15T06:15:14,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-12-15T06:15:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36391 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:15:15,590 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart, procId: 9 completed 2024-12-15T06:15:15,593 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-12-15T06:15:15,593 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:15,789 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:16,790 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:17,599 INFO [Time-limited test {}] wal.TestLogRolling(337): log.getCurrentFileName()): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:17,599 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:17,599 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:17,599 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:17,600 WARN [DataStreamer for file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta block BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK], DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]) is bad. 2024-12-15T06:15:17,600 WARN [DataStreamer for file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 block BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK], DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]) is bad. 2024-12-15T06:15:17,600 WARN [DataStreamer for file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 block BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK], DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33603,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]) is bad. 2024-12-15T06:15:17,600 WARN [PacketResponder: BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33603] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:49822 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49822 dst: /127.0.0.1:33603 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:49810 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49810 dst: /127.0.0.1:33603 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,601 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:55038 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55038 dst: /127.0.0.1:42323 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,601 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:55040 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55040 dst: /127.0.0.1:42323 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,601 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1386563235_22 at /127.0.0.1:55002 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55002 dst: /127.0.0.1:42323 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,601 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1386563235_22 at /127.0.0.1:49796 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49796 dst: /127.0.0.1:33603 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e00649{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:17,604 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6877c523{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:17,604 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:17,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@673c93a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:17,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@739a89dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:17,605 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:17,605 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:17,605 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:17,605 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164817931-172.17.0.2-1734243303529 (Datanode Uuid ff40fbb9-8ec8-4965-82d0-d858069548cd) service to localhost/127.0.0.1:41731 2024-12-15T06:15:17,606 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data3/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:17,607 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data4/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:17,607 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:17,617 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:17,621 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:17,621 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:17,621 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:17,622 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:15:17,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@480e1a2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:17,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b6260e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:17,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67025ef5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-36279-hadoop-hdfs-3_4_1-tests_jar-_-any-15584163141297034928/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:17,738 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@12b8be46{HTTP/1.1, (http/1.1)}{localhost:36279} 2024-12-15T06:15:17,738 INFO [Time-limited test {}] server.Server(415): Started @194867ms 2024-12-15T06:15:17,739 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:17,756 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1017 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1017 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:17,756 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:17,756 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:17,757 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1386563235_22 at /127.0.0.1:56364 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56364 dst: /127.0.0.1:42323 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,757 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:56374 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56374 dst: /127.0.0.1:42323 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,757 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:56376 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56376 dst: /127.0.0.1:42323 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:17,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b9c2d96{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:17,760 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1cf9d8d5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:17,760 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:17,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34816a02{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:17,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@447aae3f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:17,761 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:17,762 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164817931-172.17.0.2-1734243303529 (Datanode Uuid 45d2f94e-02b9-4a89-9da6-e7a421d06880) service to localhost/127.0.0.1:41731 2024-12-15T06:15:17,762 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:17,762 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:17,762 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data1/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:17,763 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data2/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:17,763 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:17,771 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:17,773 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:17,776 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:17,776 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:17,776 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:15:17,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52ba502e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:17,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f975f70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:17,791 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:17,821 WARN [Thread-1094 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:17,824 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8d753ce14cea0ed5 with lease ID 0x9d65686a2bfe348f: from storage DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d node DatanodeRegistration(127.0.0.1:39523, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=36789, infoSecurePort=0, ipcPort=33109, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:17,824 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8d753ce14cea0ed5 with lease ID 0x9d65686a2bfe348f: from storage DS-be4f8080-1ca9-4a8c-95c6-73f0f355e300 node DatanodeRegistration(127.0.0.1:39523, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=36789, infoSecurePort=0, ipcPort=33109, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:17,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e74ff25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-33181-hadoop-hdfs-3_4_1-tests_jar-_-any-14969409852703119684/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:17,895 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4254f91d{HTTP/1.1, (http/1.1)}{localhost:33181} 2024-12-15T06:15:17,895 INFO [Time-limited test {}] server.Server(415): Started @195024ms 2024-12-15T06:15:17,897 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:17,980 WARN [Thread-1125 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3bbee6b282a69ffa with lease ID 0x9d65686a2bfe3490: from storage DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034 node DatanodeRegistration(127.0.0.1:45421, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=33359, infoSecurePort=0, ipcPort=38561, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3bbee6b282a69ffa with lease ID 0x9d65686a2bfe3490: from storage DS-1b6e478d-72cd-4e00-b498-6ca55215448b node DatanodeRegistration(127.0.0.1:45421, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=33359, infoSecurePort=0, ipcPort=38561, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:18,791 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:18,915 INFO [Time-limited test {}] wal.TestLogRolling(349): Data Nodes restarted 2024-12-15T06:15:18,917 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-12-15T06:15:18,918 WARN [RS:0;00a5f8d66132:41173.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=5, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:18,918 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C41173%2C1734243304249:(num 1734243304634) roll requested 2024-12-15T06:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41173 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:18,918 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41173 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:55376 deadline: 1734243328917, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL 2024-12-15T06:15:18,924 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 newFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:18,924 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL 2024-12-15T06:15:18,924 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 with entries=5, filesize=2.09 KB; new WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:18,924 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33359:33359),(127.0.0.1/127.0.0.1:36789:36789)] 2024-12-15T06:15:18,924 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 is not closed yet, will try archiving it next time 2024-12-15T06:15:18,924 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:18,925 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:18,925 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:18,925 WARN [IPC Server handler 2 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1017 2024-12-15T06:15:18,925 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 after 0ms 2024-12-15T06:15:19,792 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:20,792 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:21,793 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:21,826 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1017: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-15T06:15:22,794 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:22,926 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 after 4001ms 2024-12-15T06:15:23,795 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:24,795 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:25,796 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:26,797 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:27,798 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:28,798 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:29,799 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:30,800 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:30,991 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-12-15T06:15:31,800 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:32,801 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:32,994 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:32,994 WARN [DataStreamer for file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 block BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45421,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK], DatanodeInfoWithStorage[127.0.0.1:39523,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45421,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]) is bad. 2024-12-15T06:15:32,995 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:47094 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:45421:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47094 dst: /127.0.0.1:45421 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:32,995 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:53546 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:39523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53546 dst: /127.0.0.1:39523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:32,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e74ff25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:32,997 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4254f91d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:32,997 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:32,997 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f975f70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:32,997 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52ba502e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:32,999 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:32,999 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:32,999 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164817931-172.17.0.2-1734243303529 (Datanode Uuid 45d2f94e-02b9-4a89-9da6-e7a421d06880) service to localhost/127.0.0.1:41731 2024-12-15T06:15:32,999 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:32,999 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data1/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:33,000 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data2/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:33,000 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:33,008 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:33,011 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:33,012 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:33,012 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:33,012 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:15:33,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c857dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:33,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ad0e26{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:33,129 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@590341ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-34255-hadoop-hdfs-3_4_1-tests_jar-_-any-4645175499450592886/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:33,129 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@448f943d{HTTP/1.1, (http/1.1)}{localhost:34255} 2024-12-15T06:15:33,129 INFO [Time-limited test {}] server.Server(415): Started @210258ms 2024-12-15T06:15:33,130 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:33,148 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1020 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:33,149 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-51495949_22 at /127.0.0.1:48266 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:39523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48266 dst: /127.0.0.1:39523 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:33,152 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67025ef5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:33,152 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@12b8be46{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:33,153 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:33,153 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b6260e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:33,153 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@480e1a2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:33,154 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:33,154 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164817931-172.17.0.2-1734243303529 (Datanode Uuid ff40fbb9-8ec8-4965-82d0-d858069548cd) service to localhost/127.0.0.1:41731 2024-12-15T06:15:33,154 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:33,155 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:33,155 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data3/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:33,155 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data4/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:33,155 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:33,164 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:33,168 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:33,169 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:33,169 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:33,169 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:15:33,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d8b321e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:33,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c3552d5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:33,213 WARN [Thread-1169 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:33,215 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8cf4a0395efedd0 with lease ID 0x9d65686a2bfe3491: from storage DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034 node DatanodeRegistration(127.0.0.1:38331, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=45507, infoSecurePort=0, ipcPort=45293, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:33,215 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8cf4a0395efedd0 with lease ID 0x9d65686a2bfe3491: from storage DS-1b6e478d-72cd-4e00-b498-6ca55215448b node DatanodeRegistration(127.0.0.1:38331, datanodeUuid=45d2f94e-02b9-4a89-9da6-e7a421d06880, infoPort=45507, infoSecurePort=0, ipcPort=45293, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:33,290 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6bef13f6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/java.io.tmpdir/jetty-localhost-38245-hadoop-hdfs-3_4_1-tests_jar-_-any-18299348967461698911/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:33,291 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@45ddad7c{HTTP/1.1, (http/1.1)}{localhost:38245} 2024-12-15T06:15:33,291 INFO [Time-limited test {}] server.Server(415): Started @210420ms 2024-12-15T06:15:33,292 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:33,373 WARN [Thread-1200 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:33,376 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7edefc8d0b3e5dc6 with lease ID 0x9d65686a2bfe3492: from storage DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d node DatanodeRegistration(127.0.0.1:41299, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=34021, infoSecurePort=0, ipcPort=39865, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:33,376 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7edefc8d0b3e5dc6 with lease ID 0x9d65686a2bfe3492: from storage DS-be4f8080-1ca9-4a8c-95c6-73f0f355e300 node DatanodeRegistration(127.0.0.1:41299, datanodeUuid=ff40fbb9-8ec8-4965-82d0-d858069548cd, infoPort=34021, infoSecurePort=0, ipcPort=39865, storageInfo=lv=-57;cid=testClusterID;nsid=1885965924;c=1734243303529), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:33,802 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:34,184 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:15:34,310 INFO [Time-limited test {}] wal.TestLogRolling(366): Data Nodes restarted 2024-12-15T06:15:34,311 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-12-15T06:15:34,312 WARN [RS:0;00a5f8d66132:41173.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=8, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39523,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,312 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C41173%2C1734243304249:(num 1734243318918) roll requested 2024-12-15T06:15:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41173 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39523,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,313 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.1734243334312 2024-12-15T06:15:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41173 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:55376 deadline: 1734243344312, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL 2024-12-15T06:15:34,318 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 newFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 2024-12-15T06:15:34,318 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL 2024-12-15T06:15:34,318 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 2024-12-15T06:15:34,318 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34021:34021),(127.0.0.1/127.0.0.1:45507:45507)] 2024-12-15T06:15:34,318 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39523,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,319 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 is not closed yet, will try archiving it next time 2024-12-15T06:15:34,319 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39523,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,319 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:34,319 WARN [IPC Server handler 3 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1020 2024-12-15T06:15:34,319 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 after 0ms 2024-12-15T06:15:34,400 WARN [master/00a5f8d66132:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=95, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,400 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C36391%2C1734243304201:(num 1734243304326) roll requested 2024-12-15T06:15:34,400 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,401 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36391%2C1734243304201.1734243334400 2024-12-15T06:15:34,401 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,406 WARN [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL 2024-12-15T06:15:34,406 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 with entries=92, filesize=45.98 KB; new WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243334400 2024-12-15T06:15:34,407 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45507:45507),(127.0.0.1/127.0.0.1:34021:34021)] 2024-12-15T06:15:34,407 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 is not closed yet, will try archiving it next time 2024-12-15T06:15:34,407 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,407 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:34,407 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 2024-12-15T06:15:34,407 WARN [IPC Server handler 0 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741830_1016 2024-12-15T06:15:34,408 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 after 1ms 2024-12-15T06:15:34,802 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:35,803 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:36,804 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:37,376 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1016: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-15T06:15:37,804 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:38,320 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 after 4001ms 2024-12-15T06:15:38,408 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201/00a5f8d66132%2C36391%2C1734243304201.1734243304326 after 4001ms 2024-12-15T06:15:38,484 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:15:38,486 INFO [RS-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38670, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:15:38,805 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:39,805 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:40,215 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741839_1020: GenerationStamp not matched, existing replica is blk_1073741839_1018 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-15T06:15:40,806 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:41,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 69ff2480557a57f1cd7d9a62de1d020c, had cached 0 bytes from a total of 23930 2024-12-15T06:15:41,807 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:42,807 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:43,808 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:44,808 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:45,518 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-15T06:15:45,518 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-15T06:15:45,809 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:46,338 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:46,344 DEBUG [Time-limited test {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 newFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:46,346 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:46,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45507:45507),(127.0.0.1/127.0.0.1:34021:34021)] 2024-12-15T06:15:46,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 is not closed yet, will try archiving it next time 2024-12-15T06:15:46,346 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:46,346 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:46,346 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 after 0ms 2024-12-15T06:15:46,346 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:46,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741840_1021 (size=1264) 2024-12-15T06:15:46,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741840_1021 (size=1264) 2024-12-15T06:15:46,353 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1734243305394/Put/vlen=162/seqid=0] 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [default/info:d/1734243305455/Put/vlen=9/seqid=0] 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #5: [hbase/info:d/1734243305476/Put/vlen=7/seqid=0] 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1734243305942/Put/vlen=218/seqid=0] 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [row1002/info:/1734243315596/Put/vlen=1045/seqid=0] 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243304634 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:46,354 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:46,354 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 after 0ms 2024-12-15T06:15:46,354 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:46,358 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #6: [row1003/info:/1734243328988/Put/vlen=1045/seqid=0] 2024-12-15T06:15:46,358 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #7: [row1004/info:/1734243330992/Put/vlen=1045/seqid=0] 2024-12-15T06:15:46,358 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243318918 2024-12-15T06:15:46,358 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 2024-12-15T06:15:46,358 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 2024-12-15T06:15:46,358 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 after 0ms 2024-12-15T06:15:46,358 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243334312 2024-12-15T06:15:46,361 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #9: [row1005/info:/1734243344337/Put/vlen=1045/seqid=0] 2024-12-15T06:15:46,361 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:46,361 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:46,362 WARN [IPC Server handler 0 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741842_1025 2024-12-15T06:15:46,362 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 after 1ms 2024-12-15T06:15:46,810 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:47,219 WARN [ResponseProcessor for block BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:47,219 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1386563235_22 at /127.0.0.1:52532 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:38331:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52532 dst: /127.0.0.1:38331 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38331 remote=/127.0.0.1:52532]. Total timeout mills is 60000, 59125 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:47,219 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1386563235_22 at /127.0.0.1:56602 [Receiving block BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:41299:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56602 dst: /127.0.0.1:41299 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:15:47,219 WARN [DataStreamer for file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 block BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38331,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK], DatanodeInfoWithStorage[127.0.0.1:41299,DS-66a67c48-4ddf-45c8-a42f-ca3240a5031d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38331,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]) is bad. 2024-12-15T06:15:47,223 WARN [DataStreamer for file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 block BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:47,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741842_1026 (size=85) 2024-12-15T06:15:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741842_1026 (size=85) 2024-12-15T06:15:47,810 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:48,811 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:49,812 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:50,363 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 after 4002ms 2024-12-15T06:15:50,363 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:50,367 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:50,367 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.42 KB 2024-12-15T06:15:50,368 WARN [RS_OPEN_META-regionserver/00a5f8d66132:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,368 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C41173%2C1734243304249.meta:.meta(num 1734243305002) roll requested 2024-12-15T06:15:50,368 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-15T06:15:50,368 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,368 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.meta.1734243350368.meta 2024-12-15T06:15:50,368 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 5bc9f5103f6f6ae27c9adb5cb68206ad 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-12-15T06:15:50,369 WARN [RS:0;00a5f8d66132:41173.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=11, requesting roll of WAL org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,369 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 5bc9f5103f6f6ae27c9adb5cb68206ad: 2024-12-15T06:15:50,369 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,369 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 036d6e4c75825a641cbf43445e948636 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-15T06:15:50,370 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 036d6e4c75825a641cbf43445e948636: 2024-12-15T06:15:50,370 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,373 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-15T06:15:50,373 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-15T06:15:50,373 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x42e4c0dc to 127.0.0.1:60839 2024-12-15T06:15:50,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:50,373 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-15T06:15:50,373 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1514374810, stopped=false 2024-12-15T06:15:50,373 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=00a5f8d66132,36391,1734243304201 2024-12-15T06:15:50,374 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL 2024-12-15T06:15:50,374 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta with entries=11, filesize=3.66 KB; new WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243350368.meta 2024-12-15T06:15:50,374 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34021:34021),(127.0.0.1/127.0.0.1:45507:45507)] 2024-12-15T06:15:50,374 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta is not closed yet, will try archiving it next time 2024-12-15T06:15:50,374 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 00a5f8d66132%2C41173%2C1734243304249:(num 1734243346338) roll requested 2024-12-15T06:15:50,374 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,374 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42323,DS-5bc83d9e-fab1-4be6-b9ed-c804116ac034,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,374 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta 2024-12-15T06:15:50,375 INFO [regionserver/00a5f8d66132:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C41173%2C1734243304249.1734243350374 2024-12-15T06:15:50,375 WARN [IPC Server handler 3 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta has not been closed. Lease recovery is in progress. RecoveryId = 1028 for block blk_1073741834_1015 2024-12-15T06:15:50,375 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta after 1ms 2024-12-15T06:15:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:15:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:15:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:50,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:50,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-15T06:15:50,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:50,376 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,41173,1734243304249' ***** 2024-12-15T06:15:50,376 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:15:50,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:15:50,377 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:15:50,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:15:50,377 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:15:50,377 INFO [RS:0;00a5f8d66132:41173 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:15:50,377 INFO [RS:0;00a5f8d66132:41173 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:15:50,377 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3579): Received CLOSE for 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:50,378 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3579): Received CLOSE for 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:50,378 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,41173,1734243304249 2024-12-15T06:15:50,378 DEBUG [RS:0;00a5f8d66132:41173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:50,378 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5bc9f5103f6f6ae27c9adb5cb68206ad, disabling compactions & flushes 2024-12-15T06:15:50,378 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:15:50,378 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:15:50,378 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,378 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:15:50,378 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,378 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. after waiting 0 ms 2024-12-15T06:15:50,378 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:15:50,378 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,378 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 5bc9f5103f6f6ae27c9adb5cb68206ad 1/1 column families, dataSize=4.20 KB heapSize=4.98 KB 2024-12-15T06:15:50,378 WARN [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-15T06:15:50,378 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5bc9f5103f6f6ae27c9adb5cb68206ad: 2024-12-15T06:15:50,378 ERROR [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2808): ***** ABORTING region server 00a5f8d66132,41173,1734243304249: Unrecoverable exception while closing TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. ***** org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,379 ERROR [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-15T06:15:50,379 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-15T06:15:50,380 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-15T06:15:50,380 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 5bc9f5103f6f6ae27c9adb5cb68206ad=TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad., 036d6e4c75825a641cbf43445e948636=hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636.} 2024-12-15T06:15:50,380 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3579): Received CLOSE for 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-15T06:15:50,380 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:15:50,380 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1629): Waiting on 036d6e4c75825a641cbf43445e948636, 1588230740, 5bc9f5103f6f6ae27c9adb5cb68206ad 2024-12-15T06:15:50,380 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 654950784 }, "NonHeapMemoryUsage": { "committed": 169934848, "init": 7667712, "max": -1, "used": 168047600 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:15:50,380 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740 2024-12-15T06:15:50,381 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36391 {}] master.MasterRpcServices(626): 00a5f8d66132,41173,1734243304249 reported a fatal error: ***** ABORTING region server 00a5f8d66132,41173,1734243304249: Unrecoverable exception while closing TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. ***** Cause: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.ipc.RemoteException(java.io.IOException): Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) at org.apache.hadoop.ipc.Client.call(Client.java:1529) at org.apache.hadoop.ipc.Client.call(Client.java:1426) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) at jdk.proxy2/jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) at jdk.proxy2/jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) at jdk.proxy2/jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) at jdk.proxy2/jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) at jdk.proxy2/jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) at jdk.proxy2/jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) at jdk.proxy2/jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) at jdk.proxy2/jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 036d6e4c75825a641cbf43445e948636, disabling compactions & flushes 2024-12-15T06:15:50,382 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. after waiting 0 ms 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 036d6e4c75825a641cbf43445e948636: 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5bc9f5103f6f6ae27c9adb5cb68206ad, disabling compactions & flushes 2024-12-15T06:15:50,382 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. after waiting 0 ms 2024-12-15T06:15:50,382 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,383 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 newFile=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243350374 2024-12-15T06:15:50,383 ERROR [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Memstore data size is 4304 in region TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,383 WARN [regionserver/00a5f8d66132:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL 2024-12-15T06:15:50,383 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,383 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243350374 2024-12-15T06:15:50,383 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5bc9f5103f6f6ae27c9adb5cb68206ad: 2024-12-15T06:15:50,383 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45507:45507),(127.0.0.1/127.0.0.1:34021:34021)] 2024-12-15T06:15:50,383 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1734243305585.5bc9f5103f6f6ae27c9adb5cb68206ad. 2024-12-15T06:15:50,383 DEBUG [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 is not closed yet, will try archiving it next time 2024-12-15T06:15:50,383 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,384 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-164817931-172.17.0.2-1734243303529:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-15T06:15:50,384 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:50,385 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 after 1ms 2024-12-15T06:15:50,385 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.1734243346338 to hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/oldWALs/00a5f8d66132%2C41173%2C1734243304249.1734243346338 2024-12-15T06:15:50,500 INFO [regionserver/00a5f8d66132:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-15T06:15:50,500 INFO [regionserver/00a5f8d66132:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-15T06:15:50,502 INFO [regionserver/00a5f8d66132:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:15:50,580 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:15:50,580 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(3579): Received CLOSE for 036d6e4c75825a641cbf43445e948636 2024-12-15T06:15:50,580 DEBUG [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1629): Waiting on 036d6e4c75825a641cbf43445e948636, 1588230740 2024-12-15T06:15:50,580 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 036d6e4c75825a641cbf43445e948636, disabling compactions & flushes 2024-12-15T06:15:50,580 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:15:50,580 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,580 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:15:50,580 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,580 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. after waiting 0 ms 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:15:50,581 ERROR [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Memstore data size is 78 in region hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,581 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,581 ERROR [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Memstore data size is 2972 in region hbase:meta,,1.1588230740 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 036d6e4c75825a641cbf43445e948636: 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734243305042.036d6e4c75825a641cbf43445e948636. 2024-12-15T06:15:50,581 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:15:50,581 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-15T06:15:50,781 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,41173,1734243304249; all regions closed. 2024-12-15T06:15:50,781 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249 2024-12-15T06:15:50,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741843_1027 (size=93) 2024-12-15T06:15:50,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741843_1027 (size=93) 2024-12-15T06:15:50,812 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:51,813 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:52,379 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-15T06:15:52,813 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:53,814 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:54,376 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249/00a5f8d66132%2C41173%2C1734243304249.meta.1734243305002.meta after 4002ms 2024-12-15T06:15:54,376 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/WALs/00a5f8d66132,41173,1734243304249 2024-12-15T06:15:54,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741844_1029 (size=93) 2024-12-15T06:15:54,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741844_1029 (size=93) 2024-12-15T06:15:54,379 DEBUG [RS:0;00a5f8d66132:41173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:54,380 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:15:54,380 INFO [RS:0;00a5f8d66132:41173 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-15T06:15:54,380 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:15:54,380 INFO [RS:0;00a5f8d66132:41173 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41173 2024-12-15T06:15:54,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,41173,1734243304249 2024-12-15T06:15:54,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:15:54,383 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,41173,1734243304249] 2024-12-15T06:15:54,383 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,41173,1734243304249; numProcessing=1 2024-12-15T06:15:54,385 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,41173,1734243304249 already deleted, retry=false 2024-12-15T06:15:54,385 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,41173,1734243304249 expired; onlineServers=0 2024-12-15T06:15:54,385 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,36391,1734243304201' ***** 2024-12-15T06:15:54,385 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-15T06:15:54,385 DEBUG [M:0;00a5f8d66132:36391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ec73a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:15:54,385 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,36391,1734243304201 2024-12-15T06:15:54,385 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,36391,1734243304201; all regions closed. 2024-12-15T06:15:54,385 DEBUG [M:0;00a5f8d66132:36391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:15:54,385 DEBUG [M:0;00a5f8d66132:36391 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-15T06:15:54,385 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-15T06:15:54,385 DEBUG [M:0;00a5f8d66132:36391 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-15T06:15:54,385 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243304402 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243304402,5,FailOnTimeoutGroup] 2024-12-15T06:15:54,385 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243304402 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243304402,5,FailOnTimeoutGroup] 2024-12-15T06:15:54,385 INFO [M:0;00a5f8d66132:36391 {}] hbase.ChoreService(370): Chore service for: master/00a5f8d66132:0 had [] on shutdown 2024-12-15T06:15:54,386 DEBUG [M:0;00a5f8d66132:36391 {}] master.HMaster(1733): Stopping service threads 2024-12-15T06:15:54,386 INFO [M:0;00a5f8d66132:36391 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-15T06:15:54,386 INFO [M:0;00a5f8d66132:36391 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-15T06:15:54,386 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-15T06:15:54,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-15T06:15:54,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:54,387 DEBUG [M:0;00a5f8d66132:36391 {}] zookeeper.ZKUtil(347): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-15T06:15:54,387 WARN [M:0;00a5f8d66132:36391 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-15T06:15:54,387 INFO [M:0;00a5f8d66132:36391 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-15T06:15:54,387 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:15:54,387 INFO [M:0;00a5f8d66132:36391 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-15T06:15:54,387 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:15:54,387 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:54,387 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:54,387 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:15:54,387 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:54,387 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.23 KB 2024-12-15T06:15:54,404 DEBUG [M:0;00a5f8d66132:36391 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4b2c961c0b4481497334b4608ed6939 is 82, key is hbase:meta,,1/info:regioninfo/1734243305022/Put/seqid=0 2024-12-15T06:15:54,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741845_1030 (size=5672) 2024-12-15T06:15:54,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741845_1030 (size=5672) 2024-12-15T06:15:54,410 INFO [M:0;00a5f8d66132:36391 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4b2c961c0b4481497334b4608ed6939 2024-12-15T06:15:54,431 DEBUG [M:0;00a5f8d66132:36391 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/069a22e41dd549698e191823be202b50 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734243305958/Put/seqid=0 2024-12-15T06:15:54,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741846_1031 (size=7468) 2024-12-15T06:15:54,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741846_1031 (size=7468) 2024-12-15T06:15:54,437 INFO [M:0;00a5f8d66132:36391 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.48 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/069a22e41dd549698e191823be202b50 2024-12-15T06:15:54,457 DEBUG [M:0;00a5f8d66132:36391 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d1383d33e36466d8dee7a22de06809c is 69, key is 00a5f8d66132,41173,1734243304249/rs:state/1734243304488/Put/seqid=0 2024-12-15T06:15:54,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741847_1032 (size=5156) 2024-12-15T06:15:54,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741847_1032 (size=5156) 2024-12-15T06:15:54,463 INFO [M:0;00a5f8d66132:36391 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d1383d33e36466d8dee7a22de06809c 2024-12-15T06:15:54,483 DEBUG [M:0;00a5f8d66132:36391 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/439923e58bf2481aaf3b06a84c9810c2 is 52, key is load_balancer_on/state:d/1734243305579/Put/seqid=0 2024-12-15T06:15:54,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:54,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41173-0x10094507be10001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:54,484 INFO [RS:0;00a5f8d66132:41173 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,41173,1734243304249; zookeeper connection closed. 2024-12-15T06:15:54,484 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46db0602 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46db0602 2024-12-15T06:15:54,484 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-15T06:15:54,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741848_1033 (size=5056) 2024-12-15T06:15:54,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741848_1033 (size=5056) 2024-12-15T06:15:54,489 INFO [M:0;00a5f8d66132:36391 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/439923e58bf2481aaf3b06a84c9810c2 2024-12-15T06:15:54,495 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4b2c961c0b4481497334b4608ed6939 as hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e4b2c961c0b4481497334b4608ed6939 2024-12-15T06:15:54,500 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e4b2c961c0b4481497334b4608ed6939, entries=8, sequenceid=96, filesize=5.5 K 2024-12-15T06:15:54,501 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/069a22e41dd549698e191823be202b50 as hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/069a22e41dd549698e191823be202b50 2024-12-15T06:15:54,505 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/069a22e41dd549698e191823be202b50, entries=11, sequenceid=96, filesize=7.3 K 2024-12-15T06:15:54,506 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d1383d33e36466d8dee7a22de06809c as hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d1383d33e36466d8dee7a22de06809c 2024-12-15T06:15:54,511 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d1383d33e36466d8dee7a22de06809c, entries=1, sequenceid=96, filesize=5.0 K 2024-12-15T06:15:54,512 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/439923e58bf2481aaf3b06a84c9810c2 as hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/439923e58bf2481aaf3b06a84c9810c2 2024-12-15T06:15:54,517 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41731/user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/439923e58bf2481aaf3b06a84c9810c2, entries=1, sequenceid=96, filesize=4.9 K 2024-12-15T06:15:54,518 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41040, heapSize ~49.16 KB/50344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=96, compaction requested=false 2024-12-15T06:15:54,520 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:54,520 DEBUG [M:0;00a5f8d66132:36391 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:15:54,520 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/61f77fc5-72ce-863e-e6ad-10ab36b79a6e/MasterData/WALs/00a5f8d66132,36391,1734243304201 2024-12-15T06:15:54,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41299 is added to blk_1073741841_1023 (size=757) 2024-12-15T06:15:54,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38331 is added to blk_1073741841_1023 (size=757) 2024-12-15T06:15:54,523 INFO [M:0;00a5f8d66132:36391 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-15T06:15:54,523 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:15:54,523 INFO [M:0;00a5f8d66132:36391 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36391 2024-12-15T06:15:54,525 DEBUG [M:0;00a5f8d66132:36391 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/00a5f8d66132,36391,1734243304201 already deleted, retry=false 2024-12-15T06:15:54,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:54,626 INFO [M:0;00a5f8d66132:36391 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,36391,1734243304201; zookeeper connection closed. 2024-12-15T06:15:54,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36391-0x10094507be10000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:15:54,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6bef13f6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:54,629 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@45ddad7c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:54,629 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:54,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c3552d5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:54,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d8b321e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:54,631 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:54,631 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:54,631 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164817931-172.17.0.2-1734243303529 (Datanode Uuid ff40fbb9-8ec8-4965-82d0-d858069548cd) service to localhost/127.0.0.1:41731 2024-12-15T06:15:54,632 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:54,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data3/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:54,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data4/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:54,633 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:54,635 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@590341ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:54,635 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@448f943d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:54,635 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:54,635 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11ad0e26{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:54,635 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c857dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:54,637 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:15:54,637 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:15:54,637 WARN [BP-164817931-172.17.0.2-1734243303529 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-164817931-172.17.0.2-1734243303529 (Datanode Uuid 45d2f94e-02b9-4a89-9da6-e7a421d06880) service to localhost/127.0.0.1:41731 2024-12-15T06:15:54,637 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:15:54,638 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data1/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:54,638 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/cluster_85cbbad4-54b7-4ab8-cd0b-387cbb1f994e/dfs/data/data2/current/BP-164817931-172.17.0.2-1734243303529 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:15:54,638 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:15:54,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d8de0cb{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:15:54,646 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@723e44b3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:15:54,646 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:15:54,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53b34952{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:15:54,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16bb361{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir/,STOPPED} 2024-12-15T06:15:54,653 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-15T06:15:54,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-15T06:15:54,677 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=101 (was 87) Potentially hanging thread: RS-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-27-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41731 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-27-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:41731 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:41731 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-27-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:41731 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-28-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1834838399) connection to localhost/127.0.0.1:41731 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-29-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=438 (was 428) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=53 (was 123), ProcessCount=11 (was 11), AvailableMemoryMB=4342 (was 4435) 2024-12-15T06:15:54,684 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=101, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=53, ProcessCount=11, AvailableMemoryMB=4342 2024-12-15T06:15:54,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-15T06:15:54,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.log.dir so I do NOT create it in target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5 2024-12-15T06:15:54,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d4b621ec-c58b-b1f6-8668-db2ecaed716b/hadoop.tmp.dir so I do NOT create it in target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5 2024-12-15T06:15:54,684 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d, deleteOnExit=true 2024-12-15T06:15:54,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/test.cache.data in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.tmp.dir in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-15T06:15:54,685 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:15:54,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/nfs.dump.dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/java.io.tmpdir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-15T06:15:54,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-15T06:15:54,699 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:15:54,766 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:54,770 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:54,776 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:54,776 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:54,776 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:15:54,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:54,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b401235{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:54,778 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58419cb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:54,815 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:54,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d27397b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/java.io.tmpdir/jetty-localhost-41511-hadoop-hdfs-3_4_1-tests_jar-_-any-11631367579774798918/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:15:54,892 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b02d39a{HTTP/1.1, (http/1.1)}{localhost:41511} 2024-12-15T06:15:54,893 INFO [Time-limited test {}] server.Server(415): Started @232021ms 2024-12-15T06:15:54,906 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:15:54,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-15T06:15:54,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-15T06:15:54,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-15T06:15:54,971 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:54,973 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:54,974 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:54,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:54,975 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:15:54,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d3e3e4e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:54,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47606283{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:55,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bf7f79f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/java.io.tmpdir/jetty-localhost-34153-hadoop-hdfs-3_4_1-tests_jar-_-any-5683670880703027781/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:55,090 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7edf8b05{HTTP/1.1, (http/1.1)}{localhost:34153} 2024-12-15T06:15:55,090 INFO [Time-limited test {}] server.Server(415): Started @232218ms 2024-12-15T06:15:55,091 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:55,125 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:15:55,130 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:15:55,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:15:55,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:15:55,131 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:15:55,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50d9d676{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:15:55,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5fa8cb9c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:15:55,170 WARN [Thread-1362 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data1/current/BP-276581442-172.17.0.2-1734243354717/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:55,171 WARN [Thread-1363 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data2/current/BP-276581442-172.17.0.2-1734243354717/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:55,194 WARN [Thread-1341 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84a7ba4444def463 with lease ID 0x8b5915a33f893e11: Processing first storage report for DS-9dbb93c9-e2bf-419e-873e-fb20858da06d from datanode DatanodeRegistration(127.0.0.1:41971, datanodeUuid=34b3efb1-df24-4aaa-99f8-458ca0b50841, infoPort=41805, infoSecurePort=0, ipcPort=41319, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717) 2024-12-15T06:15:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84a7ba4444def463 with lease ID 0x8b5915a33f893e11: from storage DS-9dbb93c9-e2bf-419e-873e-fb20858da06d node DatanodeRegistration(127.0.0.1:41971, datanodeUuid=34b3efb1-df24-4aaa-99f8-458ca0b50841, infoPort=41805, infoSecurePort=0, ipcPort=41319, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84a7ba4444def463 with lease ID 0x8b5915a33f893e11: Processing first storage report for DS-7db101af-6622-4827-8888-103a96538756 from datanode DatanodeRegistration(127.0.0.1:41971, datanodeUuid=34b3efb1-df24-4aaa-99f8-458ca0b50841, infoPort=41805, infoSecurePort=0, ipcPort=41319, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717) 2024-12-15T06:15:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84a7ba4444def463 with lease ID 0x8b5915a33f893e11: from storage DS-7db101af-6622-4827-8888-103a96538756 node DatanodeRegistration(127.0.0.1:41971, datanodeUuid=34b3efb1-df24-4aaa-99f8-458ca0b50841, infoPort=41805, infoSecurePort=0, ipcPort=41319, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:55,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3d2deac6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/java.io.tmpdir/jetty-localhost-44111-hadoop-hdfs-3_4_1-tests_jar-_-any-17404761854498218113/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:15:55,266 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@23af104e{HTTP/1.1, (http/1.1)}{localhost:44111} 2024-12-15T06:15:55,266 INFO [Time-limited test {}] server.Server(415): Started @232395ms 2024-12-15T06:15:55,268 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:15:55,346 WARN [Thread-1388 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data3/current/BP-276581442-172.17.0.2-1734243354717/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:55,347 WARN [Thread-1389 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data4/current/BP-276581442-172.17.0.2-1734243354717/current, will proceed with Du for space computation calculation, 2024-12-15T06:15:55,363 WARN [Thread-1377 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:15:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x177c53e718a799b9 with lease ID 0x8b5915a33f893e12: Processing first storage report for DS-8a6e3c2b-3dc9-42bd-b1f8-eff6ef2c15da from datanode DatanodeRegistration(127.0.0.1:37229, datanodeUuid=19cc24e4-efc3-4b73-847d-989c4c46ae0b, infoPort=35215, infoSecurePort=0, ipcPort=46671, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717) 2024-12-15T06:15:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x177c53e718a799b9 with lease ID 0x8b5915a33f893e12: from storage DS-8a6e3c2b-3dc9-42bd-b1f8-eff6ef2c15da node DatanodeRegistration(127.0.0.1:37229, datanodeUuid=19cc24e4-efc3-4b73-847d-989c4c46ae0b, infoPort=35215, infoSecurePort=0, ipcPort=46671, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x177c53e718a799b9 with lease ID 0x8b5915a33f893e12: Processing first storage report for DS-132d91b2-19fd-4a9a-9b36-a664974a3cfb from datanode DatanodeRegistration(127.0.0.1:37229, datanodeUuid=19cc24e4-efc3-4b73-847d-989c4c46ae0b, infoPort=35215, infoSecurePort=0, ipcPort=46671, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717) 2024-12-15T06:15:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x177c53e718a799b9 with lease ID 0x8b5915a33f893e12: from storage DS-132d91b2-19fd-4a9a-9b36-a664974a3cfb node DatanodeRegistration(127.0.0.1:37229, datanodeUuid=19cc24e4-efc3-4b73-847d-989c4c46ae0b, infoPort=35215, infoSecurePort=0, ipcPort=46671, storageInfo=lv=-57;cid=testClusterID;nsid=1892600006;c=1734243354717), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:15:55,393 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5 2024-12-15T06:15:55,395 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,396 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,396 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,396 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,396 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,397 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/zookeeper_0, clientPort=62058, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-15T06:15:55,397 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,397 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=62058 2024-12-15T06:15:55,398 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,400 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,400 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,400 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,400 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,402 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,406 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,407 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,407 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,408 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:15:55,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:15:55,411 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee with version=8 2024-12-15T06:15:55,411 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase-staging 2024-12-15T06:15:55,413 INFO [Time-limited test {}] client.ConnectionUtils(129): master/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:15:55,413 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:15:55,414 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38693 2024-12-15T06:15:55,414 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,415 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,417 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:38693 connecting to ZooKeeper ensemble=127.0.0.1:62058 2024-12-15T06:15:55,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:386930x0, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:15:55,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38693-0x100945143ea0000 connected 2024-12-15T06:15:55,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:15:55,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:15:55,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:15:55,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38693 2024-12-15T06:15:55,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38693 2024-12-15T06:15:55,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38693 2024-12-15T06:15:55,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38693 2024-12-15T06:15:55,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38693 2024-12-15T06:15:55,446 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee, hbase.cluster.distributed=false 2024-12-15T06:15:55,463 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:15:55,463 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:15:55,464 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36035 2024-12-15T06:15:55,464 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:15:55,465 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:15:55,465 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,467 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,470 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36035 connecting to ZooKeeper ensemble=127.0.0.1:62058 2024-12-15T06:15:55,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360350x0, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:15:55,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360350x0, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:15:55,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36035-0x100945143ea0001 connected 2024-12-15T06:15:55,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:15:55,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:15:55,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36035 2024-12-15T06:15:55,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36035 2024-12-15T06:15:55,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36035 2024-12-15T06:15:55,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36035 2024-12-15T06:15:55,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36035 2024-12-15T06:15:55,482 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:55,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:55,484 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:15:55,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:15:55,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,485 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:15:55,486 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00a5f8d66132,38693,1734243355412 from backup master directory 2024-12-15T06:15:55,486 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:15:55,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:55,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:15:55,487 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:15:55,487 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,500 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00a5f8d66132:38693 2024-12-15T06:15:55,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:15:55,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:15:55,503 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/hbase.id with ID: 4ec532c5-050a-4154-97cd-ac4f0c0eb962 2024-12-15T06:15:55,513 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:55,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:15:55,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:15:55,526 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:15:55,527 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-15T06:15:55,527 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:15:55,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:15:55,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:15:55,536 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store 2024-12-15T06:15:55,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:15:55,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:15:55,543 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:55,543 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:15:55,543 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:55,543 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:55,543 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:15:55,543 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:55,543 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:15:55,543 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:15:55,544 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/.initializing 2024-12-15T06:15:55,544 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/WALs/00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,546 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C38693%2C1734243355412, suffix=, logDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/WALs/00a5f8d66132,38693,1734243355412, archiveDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/oldWALs, maxLogs=10 2024-12-15T06:15:55,547 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C38693%2C1734243355412.1734243355546 2024-12-15T06:15:55,551 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/WALs/00a5f8d66132,38693,1734243355412/00a5f8d66132%2C38693%2C1734243355412.1734243355546 2024-12-15T06:15:55,551 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41805:41805),(127.0.0.1/127.0.0.1:35215:35215)] 2024-12-15T06:15:55,551 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:55,551 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:55,552 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,552 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-15T06:15:55,555 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:55,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-15T06:15:55,556 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:55,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-15T06:15:55,558 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:55,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,559 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-15T06:15:55,559 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:55,561 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,561 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,563 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-15T06:15:55,564 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:15:55,566 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:55,566 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852923, jitterRate=0.08454778790473938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-15T06:15:55,567 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:15:55,567 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-15T06:15:55,570 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bf1189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:15:55,571 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-15T06:15:55,571 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-15T06:15:55,571 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-15T06:15:55,571 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-15T06:15:55,572 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-15T06:15:55,572 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-15T06:15:55,572 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-15T06:15:55,574 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-15T06:15:55,574 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-15T06:15:55,575 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-15T06:15:55,576 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-15T06:15:55,576 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-15T06:15:55,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-15T06:15:55,577 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-15T06:15:55,578 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-15T06:15:55,579 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-15T06:15:55,580 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-15T06:15:55,581 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-15T06:15:55,582 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-15T06:15:55,583 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-15T06:15:55,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:15:55,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:15:55,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,585 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=00a5f8d66132,38693,1734243355412, sessionid=0x100945143ea0000, setting cluster-up flag (Was=false) 2024-12-15T06:15:55,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,591 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-15T06:15:55,592 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:55,599 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-15T06:15:55,599 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,38693,1734243355412 2024-12-15T06:15:55,602 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-15T06:15:55,602 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-15T06:15:55,602 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-15T06:15:55,602 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00a5f8d66132,38693,1734243355412 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00a5f8d66132:0, corePoolSize=10, maxPoolSize=10 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:15:55,603 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734243385604 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,604 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:15:55,604 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-15T06:15:55,604 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-15T06:15:55,605 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243355605,5,FailOnTimeoutGroup] 2024-12-15T06:15:55,605 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243355605,5,FailOnTimeoutGroup] 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-15T06:15:55,605 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,606 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,606 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,606 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:15:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:15:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:15:55,613 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-15T06:15:55,613 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee 2024-12-15T06:15:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:15:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:15:55,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:55,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:15:55,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:15:55,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:55,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:15:55,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:15:55,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:55,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:15:55,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:15:55,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:55,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:55,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740 2024-12-15T06:15:55,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740 2024-12-15T06:15:55,630 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:15:55,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:15:55,633 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:55,633 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826481, jitterRate=0.05092591047286987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:15:55,633 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:15:55,633 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:15:55,633 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:15:55,633 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:15:55,634 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms 2024-12-15T06:15:55,634 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:15:55,634 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:15:55,634 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:15:55,635 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:15:55,635 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-15T06:15:55,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-15T06:15:55,636 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-15T06:15:55,637 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-15T06:15:55,693 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00a5f8d66132:36035 2024-12-15T06:15:55,694 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1008): ClusterId : 4ec532c5-050a-4154-97cd-ac4f0c0eb962 2024-12-15T06:15:55,694 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:15:55,696 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:15:55,696 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:15:55,698 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:15:55,698 DEBUG [RS:0;00a5f8d66132:36035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d22ad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:15:55,699 DEBUG [RS:0;00a5f8d66132:36035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca7dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:15:55,699 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:15:55,699 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:15:55,699 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:15:55,699 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,38693,1734243355412 with isa=00a5f8d66132/172.17.0.2:36035, startcode=1734243355462 2024-12-15T06:15:55,699 DEBUG [RS:0;00a5f8d66132:36035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:15:55,701 INFO [RS-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60545, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:15:55,702 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38693 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,36035,1734243355462 2024-12-15T06:15:55,702 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38693 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:55,703 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee 2024-12-15T06:15:55,703 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:38183 2024-12-15T06:15:55,703 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:15:55,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:15:55,705 DEBUG [RS:0;00a5f8d66132:36035 {}] zookeeper.ZKUtil(111): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,36035,1734243355462 2024-12-15T06:15:55,705 WARN [RS:0;00a5f8d66132:36035 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:15:55,705 INFO [RS:0;00a5f8d66132:36035 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:15:55,705 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462 2024-12-15T06:15:55,706 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,36035,1734243355462] 2024-12-15T06:15:55,708 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:15:55,708 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:15:55,709 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:15:55,710 INFO [RS:0;00a5f8d66132:36035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:15:55,710 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,710 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:15:55,711 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:15:55,711 DEBUG [RS:0;00a5f8d66132:36035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:15:55,712 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,712 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,712 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,712 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,712 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36035,1734243355462-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:15:55,727 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:15:55,727 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,36035,1734243355462-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:55,741 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.Replication(204): 00a5f8d66132,36035,1734243355462 started 2024-12-15T06:15:55,741 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,36035,1734243355462, RpcServer on 00a5f8d66132/172.17.0.2:36035, sessionid=0x100945143ea0001 2024-12-15T06:15:55,741 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:15:55,741 DEBUG [RS:0;00a5f8d66132:36035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,36035,1734243355462 2024-12-15T06:15:55,741 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,36035,1734243355462' 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,36035,1734243355462 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,36035,1734243355462' 2024-12-15T06:15:55,742 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:15:55,743 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:15:55,743 DEBUG [RS:0;00a5f8d66132:36035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:15:55,743 INFO [RS:0;00a5f8d66132:36035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:15:55,743 INFO [RS:0;00a5f8d66132:36035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:15:55,787 WARN [00a5f8d66132:38693 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-15T06:15:55,815 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:55,845 INFO [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C36035%2C1734243355462, suffix=, logDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462, archiveDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/oldWALs, maxLogs=32 2024-12-15T06:15:55,846 INFO [RS:0;00a5f8d66132:36035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36035%2C1734243355462.1734243355846 2024-12-15T06:15:55,852 INFO [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243355846 2024-12-15T06:15:55,852 DEBUG [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41805:41805),(127.0.0.1/127.0.0.1:35215:35215)] 2024-12-15T06:15:55,909 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:15:55,910 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,910 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,910 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,910 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,927 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,927 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,927 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,928 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,928 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,929 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:55,935 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:15:56,037 DEBUG [00a5f8d66132:38693 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-15T06:15:56,038 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,039 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,36035,1734243355462, state=OPENING 2024-12-15T06:15:56,040 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-15T06:15:56,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:56,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:56,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=00a5f8d66132,36035,1734243355462}] 2024-12-15T06:15:56,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:56,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:56,195 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,195 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-15T06:15:56,197 INFO [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54030, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-15T06:15:56,201 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-15T06:15:56,201 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:15:56,203 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C36035%2C1734243355462.meta, suffix=.meta, logDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462, archiveDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/oldWALs, maxLogs=32 2024-12-15T06:15:56,204 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36035%2C1734243355462.meta.1734243356203.meta 2024-12-15T06:15:56,212 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.meta.1734243356203.meta 2024-12-15T06:15:56,212 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35215:35215),(127.0.0.1/127.0.0.1:41805:41805)] 2024-12-15T06:15:56,212 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:56,212 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-15T06:15:56,212 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-15T06:15:56,212 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-15T06:15:56,213 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-15T06:15:56,213 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:56,213 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-15T06:15:56,213 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-15T06:15:56,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:15:56,215 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:15:56,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:56,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:56,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:15:56,216 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:15:56,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:56,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:56,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:15:56,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:15:56,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:56,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:15:56,219 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740 2024-12-15T06:15:56,220 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740 2024-12-15T06:15:56,221 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:15:56,223 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:15:56,224 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=736378, jitterRate=-0.06364697217941284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:15:56,224 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:15:56,224 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734243356195 2024-12-15T06:15:56,226 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-15T06:15:56,227 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-15T06:15:56,227 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,228 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,36035,1734243355462, state=OPEN 2024-12-15T06:15:56,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:15:56,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:15:56,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:56,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:15:56,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-15T06:15:56,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=00a5f8d66132,36035,1734243355462 in 190 msec 2024-12-15T06:15:56,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-15T06:15:56,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 599 msec 2024-12-15T06:15:56,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 636 msec 2024-12-15T06:15:56,238 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734243356238, completionTime=-1 2024-12-15T06:15:56,238 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-15T06:15:56,238 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-15T06:15:56,239 DEBUG [hconnection-0x1a25e773-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:15:56,240 INFO [RS-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54046, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:15:56,241 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-15T06:15:56,241 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734243416241 2024-12-15T06:15:56,241 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734243476241 2024-12-15T06:15:56,241 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec 2024-12-15T06:15:56,246 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38693,1734243355412-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:56,246 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38693,1734243355412-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:56,246 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38693,1734243355412-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:56,246 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00a5f8d66132:38693, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:56,246 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:56,247 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-15T06:15:56,247 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:15:56,248 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-15T06:15:56,248 DEBUG [master/00a5f8d66132:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-15T06:15:56,249 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:15:56,249 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:56,250 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:15:56,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:15:56,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:15:56,259 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8b445d762040bb2b9dc66424052e2f72, NAME => 'hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee 2024-12-15T06:15:56,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:15:56,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:15:56,270 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:56,270 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 8b445d762040bb2b9dc66424052e2f72, disabling compactions & flushes 2024-12-15T06:15:56,270 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,271 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,271 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. after waiting 0 ms 2024-12-15T06:15:56,271 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,271 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,271 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8b445d762040bb2b9dc66424052e2f72: 2024-12-15T06:15:56,272 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:15:56,272 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734243356272"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243356272"}]},"ts":"1734243356272"} 2024-12-15T06:15:56,274 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:15:56,275 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:15:56,275 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243356275"}]},"ts":"1734243356275"} 2024-12-15T06:15:56,278 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-15T06:15:56,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8b445d762040bb2b9dc66424052e2f72, ASSIGN}] 2024-12-15T06:15:56,282 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8b445d762040bb2b9dc66424052e2f72, ASSIGN 2024-12-15T06:15:56,283 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8b445d762040bb2b9dc66424052e2f72, ASSIGN; state=OFFLINE, location=00a5f8d66132,36035,1734243355462; forceNewPlan=false, retain=false 2024-12-15T06:15:56,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8b445d762040bb2b9dc66424052e2f72, regionState=OPENING, regionLocation=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8b445d762040bb2b9dc66424052e2f72, server=00a5f8d66132,36035,1734243355462}] 2024-12-15T06:15:56,589 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,593 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 8b445d762040bb2b9dc66424052e2f72, NAME => 'hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:56,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:56,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,595 INFO [StoreOpener-8b445d762040bb2b9dc66424052e2f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,596 INFO [StoreOpener-8b445d762040bb2b9dc66424052e2f72-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b445d762040bb2b9dc66424052e2f72 columnFamilyName info 2024-12-15T06:15:56,596 DEBUG [StoreOpener-8b445d762040bb2b9dc66424052e2f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:56,597 INFO [StoreOpener-8b445d762040bb2b9dc66424052e2f72-1 {}] regionserver.HStore(327): Store=8b445d762040bb2b9dc66424052e2f72/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:56,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,598 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,599 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:15:56,601 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:56,602 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 8b445d762040bb2b9dc66424052e2f72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804345, jitterRate=0.02277831733226776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:15:56,602 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 8b445d762040bb2b9dc66424052e2f72: 2024-12-15T06:15:56,603 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72., pid=6, masterSystemTime=1734243356589 2024-12-15T06:15:56,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,605 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:15:56,606 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8b445d762040bb2b9dc66424052e2f72, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-15T06:15:56,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8b445d762040bb2b9dc66424052e2f72, server=00a5f8d66132,36035,1734243355462 in 171 msec 2024-12-15T06:15:56,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-15T06:15:56,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8b445d762040bb2b9dc66424052e2f72, ASSIGN in 328 msec 2024-12-15T06:15:56,612 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:15:56,612 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243356612"}]},"ts":"1734243356612"} 2024-12-15T06:15:56,613 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-15T06:15:56,616 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:15:56,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 369 msec 2024-12-15T06:15:56,649 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-15T06:15:56,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:15:56,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:56,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:15:56,655 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-15T06:15:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:15:56,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 12 msec 2024-12-15T06:15:56,677 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-15T06:15:56,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:15:56,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 11 msec 2024-12-15T06:15:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-15T06:15:56,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.217sec 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38693,1734243355412-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:15:56,704 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38693,1734243355412-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-15T06:15:56,706 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-15T06:15:56,706 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-15T06:15:56,706 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38693,1734243355412-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:15:56,784 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69396021 to 127.0.0.1:62058 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ee106f4 2024-12-15T06:15:56,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67407e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:15:56,789 DEBUG [hconnection-0x3e55405a-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:15:56,790 INFO [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54054, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:15:56,792 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=00a5f8d66132,38693,1734243355412 2024-12-15T06:15:56,792 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:15:56,796 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-15T06:15:56,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-15T06:15:56,800 INFO [RS-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43322, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-15T06:15:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-15T06:15:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-15T06:15:56,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:15:56,803 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:15:56,803 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:56,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 9 2024-12-15T06:15:56,804 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:15:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741837_1013 (size=405) 2024-12-15T06:15:56,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741837_1013 (size=405) 2024-12-15T06:15:56,813 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 04d642dd988c6d6a00809d38a37190fd, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee 2024-12-15T06:15:56,816 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741838_1014 (size=88) 2024-12-15T06:15:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741838_1014 (size=88) 2024-12-15T06:15:56,821 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:56,821 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1681): Closing 04d642dd988c6d6a00809d38a37190fd, disabling compactions & flushes 2024-12-15T06:15:56,821 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:56,821 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:56,821 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. after waiting 0 ms 2024-12-15T06:15:56,821 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:56,821 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:56,821 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:15:56,822 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:15:56,822 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1734243356822"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243356822"}]},"ts":"1734243356822"} 2024-12-15T06:15:56,824 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:15:56,825 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:15:56,825 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243356825"}]},"ts":"1734243356825"} 2024-12-15T06:15:56,827 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-12-15T06:15:56,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=04d642dd988c6d6a00809d38a37190fd, ASSIGN}] 2024-12-15T06:15:56,831 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=04d642dd988c6d6a00809d38a37190fd, ASSIGN 2024-12-15T06:15:56,832 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=04d642dd988c6d6a00809d38a37190fd, ASSIGN; state=OFFLINE, location=00a5f8d66132,36035,1734243355462; forceNewPlan=false, retain=false 2024-12-15T06:15:56,983 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=04d642dd988c6d6a00809d38a37190fd, regionState=OPENING, regionLocation=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:56,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 04d642dd988c6d6a00809d38a37190fd, server=00a5f8d66132,36035,1734243355462}] 2024-12-15T06:15:57,137 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:15:57,141 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:57,141 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 04d642dd988c6d6a00809d38a37190fd, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:15:57,142 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,142 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:15:57,142 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,142 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,143 INFO [StoreOpener-04d642dd988c6d6a00809d38a37190fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,145 INFO [StoreOpener-04d642dd988c6d6a00809d38a37190fd-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04d642dd988c6d6a00809d38a37190fd columnFamilyName info 2024-12-15T06:15:57,145 DEBUG [StoreOpener-04d642dd988c6d6a00809d38a37190fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:15:57,145 INFO [StoreOpener-04d642dd988c6d6a00809d38a37190fd-1 {}] regionserver.HStore(327): Store=04d642dd988c6d6a00809d38a37190fd/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:15:57,146 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,146 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,148 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:15:57,150 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:15:57,150 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 04d642dd988c6d6a00809d38a37190fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=857817, jitterRate=0.09077110886573792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:15:57,151 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:15:57,152 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd., pid=11, masterSystemTime=1734243357137 2024-12-15T06:15:57,154 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:57,154 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:15:57,155 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=04d642dd988c6d6a00809d38a37190fd, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,36035,1734243355462 2024-12-15T06:15:57,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-15T06:15:57,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 04d642dd988c6d6a00809d38a37190fd, server=00a5f8d66132,36035,1734243355462 in 171 msec 2024-12-15T06:15:57,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-15T06:15:57,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=04d642dd988c6d6a00809d38a37190fd, ASSIGN in 328 msec 2024-12-15T06:15:57,161 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:15:57,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243357161"}]},"ts":"1734243357161"} 2024-12-15T06:15:57,163 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-12-15T06:15:57,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:15:57,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 365 msec 2024-12-15T06:15:57,817 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:58,817 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:15:59,818 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:00,818 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:01,714 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:16:01,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,731 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,731 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,731 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,732 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,737 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:16:01,743 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-15T06:16:01,743 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-15T06:16:01,744 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-12-15T06:16:01,819 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:02,820 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:03,820 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:04,821 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:04,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-15T06:16:04,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-15T06:16:04,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-15T06:16:04,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-15T06:16:04,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:04,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-12-15T06:16:05,822 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:05,822 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta after 68045ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor238.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:16:06,806 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 9 completed 2024-12-15T06:16:06,808 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:06,808 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:06,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush hbase:namespace 2024-12-15T06:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace 2024-12-15T06:16:06,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_PREPARE 2024-12-15T06:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-15T06:16:06,822 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:06,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-15T06:16:06,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-15T06:16:06,982 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:16:06,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36035 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-12-15T06:16:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:06,984 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 8b445d762040bb2b9dc66424052e2f72 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-15T06:16:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/.tmp/info/83501f0f80674364b70f5569d658b8b0 is 45, key is default/info:d/1734243356659/Put/seqid=0 2024-12-15T06:16:07,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741839_1015 (size=5037) 2024-12-15T06:16:07,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741839_1015 (size=5037) 2024-12-15T06:16:07,007 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/.tmp/info/83501f0f80674364b70f5569d658b8b0 2024-12-15T06:16:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/.tmp/info/83501f0f80674364b70f5569d658b8b0 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/info/83501f0f80674364b70f5569d658b8b0 2024-12-15T06:16:07,018 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/info/83501f0f80674364b70f5569d658b8b0, entries=2, sequenceid=6, filesize=4.9 K 2024-12-15T06:16:07,019 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 8b445d762040bb2b9dc66424052e2f72 in 35ms, sequenceid=6, compaction requested=false 2024-12-15T06:16:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 8b445d762040bb2b9dc66424052e2f72: 2024-12-15T06:16:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:07,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13 2024-12-15T06:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster(4106): Remote procedure done, pid=13 2024-12-15T06:16:07,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-15T06:16:07,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 200 msec 2024-12-15T06:16:07,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace in 210 msec 2024-12-15T06:16:07,823 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:08,824 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:09,824 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:10,825 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:11,826 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:12,826 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:13,827 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:14,827 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:15,828 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-15T06:16:16,824 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: hbase:namespace, procId: 12 completed 2024-12-15T06:16:16,829 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:16,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-15T06:16:16,833 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-15T06:16:16,834 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-15T06:16:16,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-15T06:16:16,986 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:16:16,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36035 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-15T06:16:16,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:16,988 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing 04d642dd988c6d6a00809d38a37190fd 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-15T06:16:17,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/508246e4d008454386bfc7423d3b73c5 is 1080, key is row0001/info:/1734243376827/Put/seqid=0 2024-12-15T06:16:17,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741840_1016 (size=6033) 2024-12-15T06:16:17,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741840_1016 (size=6033) 2024-12-15T06:16:17,011 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/508246e4d008454386bfc7423d3b73c5 2024-12-15T06:16:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/508246e4d008454386bfc7423d3b73c5 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/508246e4d008454386bfc7423d3b73c5 2024-12-15T06:16:17,024 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/508246e4d008454386bfc7423d3b73c5, entries=1, sequenceid=5, filesize=5.9 K 2024-12-15T06:16:17,025 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 04d642dd988c6d6a00809d38a37190fd in 37ms, sequenceid=5, compaction requested=false 2024-12-15T06:16:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:16:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-15T06:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster(4106): Remote procedure done, pid=15 2024-12-15T06:16:17,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-12-15T06:16:17,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 192 msec 2024-12-15T06:16:17,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 198 msec 2024-12-15T06:16:17,829 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:18,830 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:19,830 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:20,831 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:21,831 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:22,832 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:23,833 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:24,833 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:25,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:16:25,834 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:26,717 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 69ff2480557a57f1cd7d9a62de1d020c, had cached 0 bytes from a total of 23930 2024-12-15T06:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-15T06:16:26,835 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 14 completed 2024-12-15T06:16:26,834 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:26,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-15T06:16:26,841 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-15T06:16:26,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-15T06:16:26,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-15T06:16:26,994 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:16:26,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36035 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-12-15T06:16:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:26,995 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing 04d642dd988c6d6a00809d38a37190fd 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-15T06:16:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/c68c26ec115947ecb933c4c784d41e1f is 1080, key is row0002/info:/1734243386835/Put/seqid=0 2024-12-15T06:16:27,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741841_1017 (size=6033) 2024-12-15T06:16:27,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741841_1017 (size=6033) 2024-12-15T06:16:27,006 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/c68c26ec115947ecb933c4c784d41e1f 2024-12-15T06:16:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/c68c26ec115947ecb933c4c784d41e1f as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/c68c26ec115947ecb933c4c784d41e1f 2024-12-15T06:16:27,018 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/c68c26ec115947ecb933c4c784d41e1f, entries=1, sequenceid=9, filesize=5.9 K 2024-12-15T06:16:27,019 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 04d642dd988c6d6a00809d38a37190fd in 24ms, sequenceid=9, compaction requested=false 2024-12-15T06:16:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:16:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17 2024-12-15T06:16:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster(4106): Remote procedure done, pid=17 2024-12-15T06:16:27,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-12-15T06:16:27,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec 2024-12-15T06:16:27,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec 2024-12-15T06:16:27,835 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:28,836 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:29,699 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:16:29,700 INFO [RS-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46202, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:16:29,836 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:30,837 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:31,837 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:32,838 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:33,839 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:34,839 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:35,840 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:36,841 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-15T06:16:36,843 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 16 completed 2024-12-15T06:16:36,845 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36035%2C1734243355462.1734243396845 2024-12-15T06:16:36,852 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243355846 with entries=13, filesize=6.41 KB; new WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243396845 2024-12-15T06:16:36,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41805:41805),(127.0.0.1/127.0.0.1:35215:35215)] 2024-12-15T06:16:36,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243355846 is not closed yet, will try archiving it next time 2024-12-15T06:16:36,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741833_1009 (size=6574) 2024-12-15T06:16:36,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741833_1009 (size=6574) 2024-12-15T06:16:36,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-15T06:16:36,858 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-15T06:16:36,858 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-15T06:16:36,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-15T06:16:37,010 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:16:37,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36035 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-12-15T06:16:37,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:37,011 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing 04d642dd988c6d6a00809d38a37190fd 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-15T06:16:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/1e91327e79fe497fa5355bac6a5066a4 is 1080, key is row0003/info:/1734243396843/Put/seqid=0 2024-12-15T06:16:37,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741843_1019 (size=6033) 2024-12-15T06:16:37,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741843_1019 (size=6033) 2024-12-15T06:16:37,022 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/1e91327e79fe497fa5355bac6a5066a4 2024-12-15T06:16:37,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/1e91327e79fe497fa5355bac6a5066a4 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/1e91327e79fe497fa5355bac6a5066a4 2024-12-15T06:16:37,033 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/1e91327e79fe497fa5355bac6a5066a4, entries=1, sequenceid=13, filesize=5.9 K 2024-12-15T06:16:37,034 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 04d642dd988c6d6a00809d38a37190fd in 23ms, sequenceid=13, compaction requested=true 2024-12-15T06:16:37,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:16:37,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:37,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-15T06:16:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster(4106): Remote procedure done, pid=19 2024-12-15T06:16:37,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-12-15T06:16:37,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec 2024-12-15T06:16:37,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 182 msec 2024-12-15T06:16:37,465 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-15T06:16:37,465 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-15T06:16:37,841 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:38,842 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:39,842 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:40,843 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:41,593 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 8b445d762040bb2b9dc66424052e2f72, had cached 0 bytes from a total of 5037 2024-12-15T06:16:41,843 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:42,142 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 04d642dd988c6d6a00809d38a37190fd, had cached 0 bytes from a total of 18099 2024-12-15T06:16:42,844 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:43,845 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:44,845 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:45,846 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:46,846 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-15T06:16:46,859 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 18 completed 2024-12-15T06:16:46,859 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:16:46,861 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:16:46,861 DEBUG [Time-limited test {}] regionserver.HStore(1540): 04d642dd988c6d6a00809d38a37190fd/info is initiating minor compaction (all files) 2024-12-15T06:16:46,861 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:16:46,861 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:46,861 INFO [Time-limited test {}] regionserver.HRegion(2351): Starting compaction of 04d642dd988c6d6a00809d38a37190fd/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:46,861 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/508246e4d008454386bfc7423d3b73c5, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/c68c26ec115947ecb933c4c784d41e1f, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/1e91327e79fe497fa5355bac6a5066a4] into tmpdir=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp, totalSize=17.7 K 2024-12-15T06:16:46,862 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 508246e4d008454386bfc7423d3b73c5, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1734243376827 2024-12-15T06:16:46,862 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting c68c26ec115947ecb933c4c784d41e1f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1734243386835 2024-12-15T06:16:46,863 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 1e91327e79fe497fa5355bac6a5066a4, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1734243396843 2024-12-15T06:16:46,875 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 04d642dd988c6d6a00809d38a37190fd#info#compaction#28 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:16:46,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/4865784cfaf1448fad3c679d323a3b25 is 1080, key is row0001/info:/1734243376827/Put/seqid=0 2024-12-15T06:16:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741844_1020 (size=8296) 2024-12-15T06:16:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741844_1020 (size=8296) 2024-12-15T06:16:46,886 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/4865784cfaf1448fad3c679d323a3b25 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/4865784cfaf1448fad3c679d323a3b25 2024-12-15T06:16:46,892 INFO [Time-limited test {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 04d642dd988c6d6a00809d38a37190fd/info of 04d642dd988c6d6a00809d38a37190fd into 4865784cfaf1448fad3c679d323a3b25(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:16:46,892 DEBUG [Time-limited test {}] regionserver.HRegion(2381): Compaction status journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:16:46,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36035%2C1734243355462.1734243406893 2024-12-15T06:16:46,900 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243396845 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243406893 2024-12-15T06:16:46,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35215:35215),(127.0.0.1/127.0.0.1:41805:41805)] 2024-12-15T06:16:46,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243396845 is not closed yet, will try archiving it next time 2024-12-15T06:16:46,900 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243355846 to hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/oldWALs/00a5f8d66132%2C36035%2C1734243355462.1734243355846 2024-12-15T06:16:46,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741842_1018 (size=2520) 2024-12-15T06:16:46,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741842_1018 (size=2520) 2024-12-15T06:16:46,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:16:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-15T06:16:46,905 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-15T06:16:46,906 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-15T06:16:46,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-15T06:16:47,058 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,36035,1734243355462 2024-12-15T06:16:47,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36035 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-15T06:16:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:47,059 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing 04d642dd988c6d6a00809d38a37190fd 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-15T06:16:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/ee6c1054e1804a718c05825523cf19c7 is 1080, key is row0000/info:/1734243406892/Put/seqid=0 2024-12-15T06:16:47,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741846_1022 (size=6033) 2024-12-15T06:16:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741846_1022 (size=6033) 2024-12-15T06:16:47,068 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/ee6c1054e1804a718c05825523cf19c7 2024-12-15T06:16:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/ee6c1054e1804a718c05825523cf19c7 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/ee6c1054e1804a718c05825523cf19c7 2024-12-15T06:16:47,079 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/ee6c1054e1804a718c05825523cf19c7, entries=1, sequenceid=18, filesize=5.9 K 2024-12-15T06:16:47,080 INFO [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 04d642dd988c6d6a00809d38a37190fd in 21ms, sequenceid=18, compaction requested=false 2024-12-15T06:16:47,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:16:47,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:47,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21 2024-12-15T06:16:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.HMaster(4106): Remote procedure done, pid=21 2024-12-15T06:16:47,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20 2024-12-15T06:16:47,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 175 msec 2024-12-15T06:16:47,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 180 msec 2024-12-15T06:16:47,847 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:48,848 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:49,848 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:50,849 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:51,849 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:52,850 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:53,850 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:54,851 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:55,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:16:55,852 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:56,257 DEBUG [master/00a5f8d66132:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-15T06:16:56,257 DEBUG [master/00a5f8d66132:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 8b445d762040bb2b9dc66424052e2f72 changed from -1.0 to 0.0, refreshing cache 2024-12-15T06:16:56,852 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38693 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-15T06:16:56,906 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 20 completed 2024-12-15T06:16:56,909 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C36035%2C1734243355462.1734243416908 2024-12-15T06:16:56,915 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243406893 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243416908 2024-12-15T06:16:56,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41805:41805),(127.0.0.1/127.0.0.1:35215:35215)] 2024-12-15T06:16:56,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243406893 is not closed yet, will try archiving it next time 2024-12-15T06:16:56,916 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-15T06:16:56,916 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462/00a5f8d66132%2C36035%2C1734243355462.1734243396845 to hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/oldWALs/00a5f8d66132%2C36035%2C1734243355462.1734243396845 2024-12-15T06:16:56,916 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-15T06:16:56,916 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69396021 to 127.0.0.1:62058 2024-12-15T06:16:56,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:16:56,916 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-15T06:16:56,916 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1115789326, stopped=false 2024-12-15T06:16:56,916 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=00a5f8d66132,38693,1734243355412 2024-12-15T06:16:56,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741845_1021 (size=2026) 2024-12-15T06:16:56,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741845_1021 (size=2026) 2024-12-15T06:16:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:16:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:16:56,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-15T06:16:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:56,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:16:56,922 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,36035,1734243355462' ***** 2024-12-15T06:16:56,922 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:16:56,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:16:56,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:16:56,922 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:16:56,922 INFO [RS:0;00a5f8d66132:36035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:16:56,922 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:16:56,922 INFO [RS:0;00a5f8d66132:36035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(3579): Received CLOSE for 04d642dd988c6d6a00809d38a37190fd 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(3579): Received CLOSE for 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,36035,1734243355462 2024-12-15T06:16:56,923 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 04d642dd988c6d6a00809d38a37190fd, disabling compactions & flushes 2024-12-15T06:16:56,923 DEBUG [RS:0;00a5f8d66132:36035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:16:56,923 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:56,923 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:56,923 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. after waiting 0 ms 2024-12-15T06:16:56,923 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:16:56,923 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 04d642dd988c6d6a00809d38a37190fd 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-15T06:16:56,923 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:16:56,924 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-15T06:16:56,924 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1603): Online Regions={04d642dd988c6d6a00809d38a37190fd=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd., 1588230740=hbase:meta,,1.1588230740, 8b445d762040bb2b9dc66424052e2f72=hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72.} 2024-12-15T06:16:56,924 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:16:56,924 DEBUG [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1629): Waiting on 04d642dd988c6d6a00809d38a37190fd, 1588230740, 8b445d762040bb2b9dc66424052e2f72 2024-12-15T06:16:56,924 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:16:56,924 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:16:56,924 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:16:56,924 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:16:56,924 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=3.05 KB heapSize=5.55 KB 2024-12-15T06:16:56,929 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/9224f12a4f5844c88c7505be971cc5bb is 1080, key is row0001/info:/1734243416907/Put/seqid=0 2024-12-15T06:16:56,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741848_1024 (size=6033) 2024-12-15T06:16:56,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741848_1024 (size=6033) 2024-12-15T06:16:56,938 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/9224f12a4f5844c88c7505be971cc5bb 2024-12-15T06:16:56,942 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/.tmp/info/69e42b8cf3b741208c06feca4903f4c8 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd./info:regioninfo/1734243357155/Put/seqid=0 2024-12-15T06:16:56,945 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/.tmp/info/9224f12a4f5844c88c7505be971cc5bb as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/9224f12a4f5844c88c7505be971cc5bb 2024-12-15T06:16:56,950 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/9224f12a4f5844c88c7505be971cc5bb, entries=1, sequenceid=22, filesize=5.9 K 2024-12-15T06:16:56,951 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 04d642dd988c6d6a00809d38a37190fd in 28ms, sequenceid=22, compaction requested=true 2024-12-15T06:16:56,954 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/508246e4d008454386bfc7423d3b73c5, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/c68c26ec115947ecb933c4c784d41e1f, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/1e91327e79fe497fa5355bac6a5066a4] to archive 2024-12-15T06:16:56,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741849_1025 (size=8430) 2024-12-15T06:16:56,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741849_1025 (size=8430) 2024-12-15T06:16:56,956 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-15T06:16:56,956 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.79 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/.tmp/info/69e42b8cf3b741208c06feca4903f4c8 2024-12-15T06:16:56,958 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/508246e4d008454386bfc7423d3b73c5 to hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/508246e4d008454386bfc7423d3b73c5 2024-12-15T06:16:56,958 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/c68c26ec115947ecb933c4c784d41e1f to hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/c68c26ec115947ecb933c4c784d41e1f 2024-12-15T06:16:56,961 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/1e91327e79fe497fa5355bac6a5066a4 to hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/info/1e91327e79fe497fa5355bac6a5066a4 2024-12-15T06:16:56,965 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/04d642dd988c6d6a00809d38a37190fd/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-12-15T06:16:56,966 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:56,966 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 04d642dd988c6d6a00809d38a37190fd: 2024-12-15T06:16:56,966 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734243356801.04d642dd988c6d6a00809d38a37190fd. 2024-12-15T06:16:56,966 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8b445d762040bb2b9dc66424052e2f72, disabling compactions & flushes 2024-12-15T06:16:56,966 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:56,966 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:56,966 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. after waiting 0 ms 2024-12-15T06:16:56,966 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:56,971 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/namespace/8b445d762040bb2b9dc66424052e2f72/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-15T06:16:56,971 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:56,972 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8b445d762040bb2b9dc66424052e2f72: 2024-12-15T06:16:56,972 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734243356247.8b445d762040bb2b9dc66424052e2f72. 2024-12-15T06:16:56,978 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/.tmp/table/aca990e1c4ad41cbb8b212d20277c278 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1734243357161/Put/seqid=0 2024-12-15T06:16:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741850_1026 (size=5532) 2024-12-15T06:16:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741850_1026 (size=5532) 2024-12-15T06:16:56,983 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=264 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/.tmp/table/aca990e1c4ad41cbb8b212d20277c278 2024-12-15T06:16:56,989 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/.tmp/info/69e42b8cf3b741208c06feca4903f4c8 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/info/69e42b8cf3b741208c06feca4903f4c8 2024-12-15T06:16:56,994 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/info/69e42b8cf3b741208c06feca4903f4c8, entries=20, sequenceid=14, filesize=8.2 K 2024-12-15T06:16:56,994 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/.tmp/table/aca990e1c4ad41cbb8b212d20277c278 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/table/aca990e1c4ad41cbb8b212d20277c278 2024-12-15T06:16:57,000 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/table/aca990e1c4ad41cbb8b212d20277c278, entries=4, sequenceid=14, filesize=5.4 K 2024-12-15T06:16:57,001 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~3.05 KB/3122, heapSize ~5.27 KB/5400, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=14, compaction requested=false 2024-12-15T06:16:57,005 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1 2024-12-15T06:16:57,005 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-15T06:16:57,005 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:16:57,005 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:16:57,005 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-15T06:16:57,124 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,36035,1734243355462; all regions closed. 2024-12-15T06:16:57,125 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462 2024-12-15T06:16:57,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741834_1010 (size=4570) 2024-12-15T06:16:57,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741834_1010 (size=4570) 2024-12-15T06:16:57,129 DEBUG [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/oldWALs 2024-12-15T06:16:57,129 INFO [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C36035%2C1734243355462.meta:.meta(num 1734243356203) 2024-12-15T06:16:57,129 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/WALs/00a5f8d66132,36035,1734243355462 2024-12-15T06:16:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741847_1023 (size=1545) 2024-12-15T06:16:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741847_1023 (size=1545) 2024-12-15T06:16:57,134 DEBUG [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(1071): Moved 2 WAL file(s) to /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/oldWALs 2024-12-15T06:16:57,134 INFO [RS:0;00a5f8d66132:36035 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C36035%2C1734243355462:(num 1734243416908) 2024-12-15T06:16:57,134 DEBUG [RS:0;00a5f8d66132:36035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:16:57,134 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:16:57,135 INFO [RS:0;00a5f8d66132:36035 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-15T06:16:57,135 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:16:57,135 INFO [RS:0;00a5f8d66132:36035 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36035 2024-12-15T06:16:57,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,36035,1734243355462 2024-12-15T06:16:57,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:16:57,138 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,36035,1734243355462] 2024-12-15T06:16:57,138 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,36035,1734243355462; numProcessing=1 2024-12-15T06:16:57,140 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,36035,1734243355462 already deleted, retry=false 2024-12-15T06:16:57,140 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,36035,1734243355462 expired; onlineServers=0 2024-12-15T06:16:57,140 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,38693,1734243355412' ***** 2024-12-15T06:16:57,140 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-15T06:16:57,140 DEBUG [M:0;00a5f8d66132:38693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8bd78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:16:57,140 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,38693,1734243355412 2024-12-15T06:16:57,140 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,38693,1734243355412; all regions closed. 2024-12-15T06:16:57,140 DEBUG [M:0;00a5f8d66132:38693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:16:57,140 DEBUG [M:0;00a5f8d66132:38693 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-15T06:16:57,140 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-15T06:16:57,140 DEBUG [M:0;00a5f8d66132:38693 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-15T06:16:57,140 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243355605 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243355605,5,FailOnTimeoutGroup] 2024-12-15T06:16:57,140 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243355605 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243355605,5,FailOnTimeoutGroup] 2024-12-15T06:16:57,141 INFO [M:0;00a5f8d66132:38693 {}] hbase.ChoreService(370): Chore service for: master/00a5f8d66132:0 had [] on shutdown 2024-12-15T06:16:57,141 DEBUG [M:0;00a5f8d66132:38693 {}] master.HMaster(1733): Stopping service threads 2024-12-15T06:16:57,141 INFO [M:0;00a5f8d66132:38693 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-15T06:16:57,141 INFO [M:0;00a5f8d66132:38693 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-15T06:16:57,141 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-15T06:16:57,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-15T06:16:57,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:57,142 DEBUG [M:0;00a5f8d66132:38693 {}] zookeeper.ZKUtil(347): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-15T06:16:57,142 WARN [M:0;00a5f8d66132:38693 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-15T06:16:57,142 INFO [M:0;00a5f8d66132:38693 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-15T06:16:57,142 INFO [M:0;00a5f8d66132:38693 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-15T06:16:57,142 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:16:57,142 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:16:57,142 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:57,142 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:57,142 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:16:57,142 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:57,143 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=65.05 KB heapSize=81.66 KB 2024-12-15T06:16:57,159 DEBUG [M:0;00a5f8d66132:38693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbd0a976f73a487498a1b847a8fa3ce9 is 82, key is hbase:meta,,1/info:regioninfo/1734243356227/Put/seqid=0 2024-12-15T06:16:57,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741851_1027 (size=5672) 2024-12-15T06:16:57,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741851_1027 (size=5672) 2024-12-15T06:16:57,164 INFO [M:0;00a5f8d66132:38693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbd0a976f73a487498a1b847a8fa3ce9 2024-12-15T06:16:57,184 DEBUG [M:0;00a5f8d66132:38693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf2dbd9b78a345b5a5c892a0ab0429bc is 798, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734243357167/Put/seqid=0 2024-12-15T06:16:57,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741852_1028 (size=8353) 2024-12-15T06:16:57,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741852_1028 (size=8353) 2024-12-15T06:16:57,190 INFO [M:0;00a5f8d66132:38693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.45 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf2dbd9b78a345b5a5c892a0ab0429bc 2024-12-15T06:16:57,194 INFO [M:0;00a5f8d66132:38693 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf2dbd9b78a345b5a5c892a0ab0429bc 2024-12-15T06:16:57,208 DEBUG [M:0;00a5f8d66132:38693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5fe360d4d27f4d47a126f158df594696 is 69, key is 00a5f8d66132,36035,1734243355462/rs:state/1734243355702/Put/seqid=0 2024-12-15T06:16:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741853_1029 (size=5156) 2024-12-15T06:16:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741853_1029 (size=5156) 2024-12-15T06:16:57,215 INFO [M:0;00a5f8d66132:38693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5fe360d4d27f4d47a126f158df594696 2024-12-15T06:16:57,234 DEBUG [M:0;00a5f8d66132:38693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/993b85686831492fac2a1f45d2ee686a is 52, key is load_balancer_on/state:d/1734243356794/Put/seqid=0 2024-12-15T06:16:57,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:16:57,239 INFO [RS:0;00a5f8d66132:36035 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,36035,1734243355462; zookeeper connection closed. 2024-12-15T06:16:57,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36035-0x100945143ea0001, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:16:57,239 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7fa342ce {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7fa342ce 2024-12-15T06:16:57,239 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-15T06:16:57,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741854_1030 (size=5056) 2024-12-15T06:16:57,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741854_1030 (size=5056) 2024-12-15T06:16:57,240 INFO [M:0;00a5f8d66132:38693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/993b85686831492fac2a1f45d2ee686a 2024-12-15T06:16:57,245 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbd0a976f73a487498a1b847a8fa3ce9 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbd0a976f73a487498a1b847a8fa3ce9 2024-12-15T06:16:57,250 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbd0a976f73a487498a1b847a8fa3ce9, entries=8, sequenceid=184, filesize=5.5 K 2024-12-15T06:16:57,251 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf2dbd9b78a345b5a5c892a0ab0429bc as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf2dbd9b78a345b5a5c892a0ab0429bc 2024-12-15T06:16:57,255 INFO [M:0;00a5f8d66132:38693 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf2dbd9b78a345b5a5c892a0ab0429bc 2024-12-15T06:16:57,255 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf2dbd9b78a345b5a5c892a0ab0429bc, entries=21, sequenceid=184, filesize=8.2 K 2024-12-15T06:16:57,256 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5fe360d4d27f4d47a126f158df594696 as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5fe360d4d27f4d47a126f158df594696 2024-12-15T06:16:57,260 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5fe360d4d27f4d47a126f158df594696, entries=1, sequenceid=184, filesize=5.0 K 2024-12-15T06:16:57,261 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/993b85686831492fac2a1f45d2ee686a as hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/993b85686831492fac2a1f45d2ee686a 2024-12-15T06:16:57,265 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38183/user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/993b85686831492fac2a1f45d2ee686a, entries=1, sequenceid=184, filesize=4.9 K 2024-12-15T06:16:57,266 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(3040): Finished flush of dataSize ~65.05 KB/66610, heapSize ~81.60 KB/83560, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=184, compaction requested=false 2024-12-15T06:16:57,268 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:57,268 DEBUG [M:0;00a5f8d66132:38693 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:16:57,268 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/711af01e-d401-a97d-0755-8488ca34b5ee/MasterData/WALs/00a5f8d66132,38693,1734243355412 2024-12-15T06:16:57,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37229 is added to blk_1073741830_1006 (size=79131) 2024-12-15T06:16:57,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741830_1006 (size=79131) 2024-12-15T06:16:57,271 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:16:57,271 INFO [M:0;00a5f8d66132:38693 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-15T06:16:57,271 INFO [M:0;00a5f8d66132:38693 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38693 2024-12-15T06:16:57,272 DEBUG [M:0;00a5f8d66132:38693 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/00a5f8d66132,38693,1734243355412 already deleted, retry=false 2024-12-15T06:16:57,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:16:57,374 INFO [M:0;00a5f8d66132:38693 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,38693,1734243355412; zookeeper connection closed. 2024-12-15T06:16:57,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38693-0x100945143ea0000, quorum=127.0.0.1:62058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:16:57,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3d2deac6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:16:57,377 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@23af104e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:16:57,377 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:16:57,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5fa8cb9c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:16:57,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50d9d676{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir/,STOPPED} 2024-12-15T06:16:57,379 WARN [BP-276581442-172.17.0.2-1734243354717 heartbeating to localhost/127.0.0.1:38183 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:16:57,379 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:16:57,379 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:16:57,379 WARN [BP-276581442-172.17.0.2-1734243354717 heartbeating to localhost/127.0.0.1:38183 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-276581442-172.17.0.2-1734243354717 (Datanode Uuid 19cc24e4-efc3-4b73-847d-989c4c46ae0b) service to localhost/127.0.0.1:38183 2024-12-15T06:16:57,379 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data3/current/BP-276581442-172.17.0.2-1734243354717 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:16:57,380 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data4/current/BP-276581442-172.17.0.2-1734243354717 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:16:57,380 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:16:57,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bf7f79f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:16:57,382 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7edf8b05{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:16:57,382 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:16:57,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47606283{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:16:57,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d3e3e4e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir/,STOPPED} 2024-12-15T06:16:57,384 WARN [BP-276581442-172.17.0.2-1734243354717 heartbeating to localhost/127.0.0.1:38183 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:16:57,384 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:16:57,384 WARN [BP-276581442-172.17.0.2-1734243354717 heartbeating to localhost/127.0.0.1:38183 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-276581442-172.17.0.2-1734243354717 (Datanode Uuid 34b3efb1-df24-4aaa-99f8-458ca0b50841) service to localhost/127.0.0.1:38183 2024-12-15T06:16:57,384 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:16:57,384 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data1/current/BP-276581442-172.17.0.2-1734243354717 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:16:57,385 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/cluster_642a5d59-c628-7c88-d292-fca8e267520d/dfs/data/data2/current/BP-276581442-172.17.0.2-1734243354717 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:16:57,385 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:16:57,391 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d27397b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:16:57,391 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b02d39a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:16:57,391 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:16:57,392 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58419cb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:16:57,392 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b401235{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir/,STOPPED} 2024-12-15T06:16:57,398 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-15T06:16:57,416 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-15T06:16:57,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=114 (was 101) - Thread LEAK? -, OpenFileDescriptor=464 (was 438) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=38 (was 53), ProcessCount=11 (was 11), AvailableMemoryMB=4195 (was 4342) 2024-12-15T06:16:57,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=115, OpenFileDescriptor=464, MaxFileDescriptor=1048576, SystemLoadAverage=38, ProcessCount=11, AvailableMemoryMB=4195 2024-12-15T06:16:57,430 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.log.dir so I do NOT create it in target/test-data/d0185466-da10-beb1-7034-be2eff28b345 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/51341376-c8f0-1752-f60b-7992ba03afc5/hadoop.tmp.dir so I do NOT create it in target/test-data/d0185466-da10-beb1-7034-be2eff28b345 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e, deleteOnExit=true 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/test.cache.data in system properties and HBase conf 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.tmp.dir in system properties and HBase conf 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir in system properties and HBase conf 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-15T06:16:57,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-15T06:16:57,431 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/nfs.dump.dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/java.io.tmpdir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-15T06:16:57,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-15T06:16:57,446 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:16:57,500 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:16:57,504 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:16:57,505 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:16:57,505 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:16:57,506 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:16:57,506 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:16:57,507 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@753c9dd9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:16:57,507 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d22749a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:16:57,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@342c9251{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/java.io.tmpdir/jetty-localhost-41909-hadoop-hdfs-3_4_1-tests_jar-_-any-5428184390150966034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:16:57,623 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@50679d28{HTTP/1.1, (http/1.1)}{localhost:41909} 2024-12-15T06:16:57,623 INFO [Time-limited test {}] server.Server(415): Started @294752ms 2024-12-15T06:16:57,636 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:16:57,684 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:16:57,688 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:16:57,689 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:16:57,689 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:16:57,689 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:16:57,690 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d6992ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:16:57,690 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26c4edf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:16:57,714 INFO [regionserver/00a5f8d66132:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:16:57,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30633a18{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/java.io.tmpdir/jetty-localhost-44023-hadoop-hdfs-3_4_1-tests_jar-_-any-15759722856632810828/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:16:57,809 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5334bcb3{HTTP/1.1, (http/1.1)}{localhost:44023} 2024-12-15T06:16:57,809 INFO [Time-limited test {}] server.Server(415): Started @294938ms 2024-12-15T06:16:57,810 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:16:57,840 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:16:57,844 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:16:57,845 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:16:57,845 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:16:57,845 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:16:57,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d35a3c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:16:57,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36c195b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:16:57,853 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:57,904 WARN [Thread-1688 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data2/current/BP-906873066-172.17.0.2-1734243417453/current, will proceed with Du for space computation calculation, 2024-12-15T06:16:57,904 WARN [Thread-1687 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data1/current/BP-906873066-172.17.0.2-1734243417453/current, will proceed with Du for space computation calculation, 2024-12-15T06:16:57,927 WARN [Thread-1666 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:16:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61936c459dd2dcd8 with lease ID 0x2c4e64c1766e0cc4: Processing first storage report for DS-daef2b57-89e8-429e-a7f0-1a1ca7f51be8 from datanode DatanodeRegistration(127.0.0.1:34657, datanodeUuid=0d054f23-d1b4-4182-8ac0-d2880884fe10, infoPort=44229, infoSecurePort=0, ipcPort=42419, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453) 2024-12-15T06:16:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61936c459dd2dcd8 with lease ID 0x2c4e64c1766e0cc4: from storage DS-daef2b57-89e8-429e-a7f0-1a1ca7f51be8 node DatanodeRegistration(127.0.0.1:34657, datanodeUuid=0d054f23-d1b4-4182-8ac0-d2880884fe10, infoPort=44229, infoSecurePort=0, ipcPort=42419, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:16:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61936c459dd2dcd8 with lease ID 0x2c4e64c1766e0cc4: Processing first storage report for DS-d776ef2d-2d9c-4d62-a276-9bde4aae6c1b from datanode DatanodeRegistration(127.0.0.1:34657, datanodeUuid=0d054f23-d1b4-4182-8ac0-d2880884fe10, infoPort=44229, infoSecurePort=0, ipcPort=42419, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453) 2024-12-15T06:16:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61936c459dd2dcd8 with lease ID 0x2c4e64c1766e0cc4: from storage DS-d776ef2d-2d9c-4d62-a276-9bde4aae6c1b node DatanodeRegistration(127.0.0.1:34657, datanodeUuid=0d054f23-d1b4-4182-8ac0-d2880884fe10, infoPort=44229, infoSecurePort=0, ipcPort=42419, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:16:57,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68e91904{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/java.io.tmpdir/jetty-localhost-39033-hadoop-hdfs-3_4_1-tests_jar-_-any-16034102228872254924/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:16:57,968 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3301755{HTTP/1.1, (http/1.1)}{localhost:39033} 2024-12-15T06:16:57,968 INFO [Time-limited test {}] server.Server(415): Started @295096ms 2024-12-15T06:16:57,969 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:16:58,049 WARN [Thread-1714 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data4/current/BP-906873066-172.17.0.2-1734243417453/current, will proceed with Du for space computation calculation, 2024-12-15T06:16:58,049 WARN [Thread-1713 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data3/current/BP-906873066-172.17.0.2-1734243417453/current, will proceed with Du for space computation calculation, 2024-12-15T06:16:58,067 WARN [Thread-1702 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:16:58,069 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfec113a843cffc3e with lease ID 0x2c4e64c1766e0cc5: Processing first storage report for DS-408b763d-a770-4bef-b074-6ac88f2330d2 from datanode DatanodeRegistration(127.0.0.1:42333, datanodeUuid=82144eaa-f56a-4e84-9e02-3126f4b854e8, infoPort=38289, infoSecurePort=0, ipcPort=35049, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453) 2024-12-15T06:16:58,069 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfec113a843cffc3e with lease ID 0x2c4e64c1766e0cc5: from storage DS-408b763d-a770-4bef-b074-6ac88f2330d2 node DatanodeRegistration(127.0.0.1:42333, datanodeUuid=82144eaa-f56a-4e84-9e02-3126f4b854e8, infoPort=38289, infoSecurePort=0, ipcPort=35049, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:16:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfec113a843cffc3e with lease ID 0x2c4e64c1766e0cc5: Processing first storage report for DS-703e80ee-e5c7-4036-bdec-6b51cb4a4175 from datanode DatanodeRegistration(127.0.0.1:42333, datanodeUuid=82144eaa-f56a-4e84-9e02-3126f4b854e8, infoPort=38289, infoSecurePort=0, ipcPort=35049, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453) 2024-12-15T06:16:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfec113a843cffc3e with lease ID 0x2c4e64c1766e0cc5: from storage DS-703e80ee-e5c7-4036-bdec-6b51cb4a4175 node DatanodeRegistration(127.0.0.1:42333, datanodeUuid=82144eaa-f56a-4e84-9e02-3126f4b854e8, infoPort=38289, infoSecurePort=0, ipcPort=35049, storageInfo=lv=-57;cid=testClusterID;nsid=2096599169;c=1734243417453), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:16:58,091 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345 2024-12-15T06:16:58,093 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/zookeeper_0, clientPort=52277, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-15T06:16:58,094 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=52277 2024-12-15T06:16:58,094 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,096 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:16:58,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:16:58,106 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5 with version=8 2024-12-15T06:16:58,106 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase-staging 2024-12-15T06:16:58,108 INFO [Time-limited test {}] client.ConnectionUtils(129): master/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:16:58,108 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:16:58,109 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:39109 2024-12-15T06:16:58,109 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,110 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,113 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:39109 connecting to ZooKeeper ensemble=127.0.0.1:52277 2024-12-15T06:16:58,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391090x0, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:16:58,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39109-0x100945238d30000 connected 2024-12-15T06:16:58,131 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:16:58,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:16:58,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:16:58,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39109 2024-12-15T06:16:58,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39109 2024-12-15T06:16:58,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39109 2024-12-15T06:16:58,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39109 2024-12-15T06:16:58,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39109 2024-12-15T06:16:58,134 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5, hbase.cluster.distributed=false 2024-12-15T06:16:58,154 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:16:58,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:16:58,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:16:58,154 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:16:58,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:16:58,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:16:58,154 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:16:58,155 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:16:58,155 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:37021 2024-12-15T06:16:58,156 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:16:58,157 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:16:58,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,160 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,163 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37021 connecting to ZooKeeper ensemble=127.0.0.1:52277 2024-12-15T06:16:58,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370210x0, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:16:58,166 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37021-0x100945238d30001 connected 2024-12-15T06:16:58,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:16:58,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:16:58,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:16:58,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37021 2024-12-15T06:16:58,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37021 2024-12-15T06:16:58,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37021 2024-12-15T06:16:58,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37021 2024-12-15T06:16:58,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37021 2024-12-15T06:16:58,174 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:16:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:16:58,177 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:16:58,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:16:58,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,178 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:16:58,179 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00a5f8d66132,39109,1734243418107 from backup master directory 2024-12-15T06:16:58,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:16:58,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:16:58,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:16:58,180 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:16:58,180 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:16:58,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:16:58,191 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/hbase.id with ID: a6524d16-1e79-42ad-a7f2-328a2e04216d 2024-12-15T06:16:58,192 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00a5f8d66132:39109 2024-12-15T06:16:58,202 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:58,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:16:58,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:16:58,212 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:16:58,212 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-15T06:16:58,213 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:16:58,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:16:58,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:16:58,221 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store 2024-12-15T06:16:58,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:16:58,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:16:58,228 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:58,228 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:16:58,228 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:58,228 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:58,228 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:16:58,228 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:58,228 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:16:58,229 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:16:58,229 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/.initializing 2024-12-15T06:16:58,229 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/WALs/00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,232 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C39109%2C1734243418107, suffix=, logDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/WALs/00a5f8d66132,39109,1734243418107, archiveDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/oldWALs, maxLogs=10 2024-12-15T06:16:58,232 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C39109%2C1734243418107.1734243418232 2024-12-15T06:16:58,237 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/WALs/00a5f8d66132,39109,1734243418107/00a5f8d66132%2C39109%2C1734243418107.1734243418232 2024-12-15T06:16:58,238 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44229:44229),(127.0.0.1/127.0.0.1:38289:38289)] 2024-12-15T06:16:58,238 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:16:58,238 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:58,238 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,238 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-15T06:16:58,241 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-15T06:16:58,242 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:16:58,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-15T06:16:58,244 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:16:58,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,245 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-15T06:16:58,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:16:58,246 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,247 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,248 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-15T06:16:58,249 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:16:58,251 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:16:58,251 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=882843, jitterRate=0.12259380519390106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-15T06:16:58,252 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:16:58,252 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-15T06:16:58,255 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66496727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:16:58,256 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-15T06:16:58,256 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-15T06:16:58,256 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-15T06:16:58,256 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-15T06:16:58,257 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-15T06:16:58,257 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-15T06:16:58,257 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-15T06:16:58,259 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-15T06:16:58,259 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-15T06:16:58,261 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-15T06:16:58,261 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-15T06:16:58,261 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-15T06:16:58,263 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-15T06:16:58,263 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-15T06:16:58,263 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-15T06:16:58,264 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-15T06:16:58,265 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-15T06:16:58,266 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-15T06:16:58,267 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-15T06:16:58,268 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-15T06:16:58,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:16:58,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:16:58,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,270 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=00a5f8d66132,39109,1734243418107, sessionid=0x100945238d30000, setting cluster-up flag (Was=false) 2024-12-15T06:16:58,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,277 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-15T06:16:58,278 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,285 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-15T06:16:58,286 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,39109,1734243418107 2024-12-15T06:16:58,288 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-15T06:16:58,288 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-15T06:16:58,288 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00a5f8d66132,39109,1734243418107 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00a5f8d66132:0, corePoolSize=10, maxPoolSize=10 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:16:58,289 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734243448290 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-15T06:16:58,290 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,290 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:16:58,290 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-15T06:16:58,291 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-15T06:16:58,291 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-15T06:16:58,291 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-15T06:16:58,292 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,292 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-15T06:16:58,292 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-15T06:16:58,292 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:16:58,292 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243418292,5,FailOnTimeoutGroup] 2024-12-15T06:16:58,292 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243418292,5,FailOnTimeoutGroup] 2024-12-15T06:16:58,292 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,292 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-15T06:16:58,292 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,292 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:16:58,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:16:58,299 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-15T06:16:58,300 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5 2024-12-15T06:16:58,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:16:58,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:16:58,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:58,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:16:58,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:16:58,309 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:16:58,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:16:58,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:16:58,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:16:58,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,313 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740 2024-12-15T06:16:58,313 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740 2024-12-15T06:16:58,314 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:16:58,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:16:58,317 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:16:58,317 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=691245, jitterRate=-0.12103700637817383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:16:58,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:16:58,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:16:58,318 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:16:58,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:16:58,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:16:58,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:16:58,318 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:16:58,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:16:58,319 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:16:58,319 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-15T06:16:58,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-15T06:16:58,320 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-15T06:16:58,321 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-15T06:16:58,386 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00a5f8d66132:37021 2024-12-15T06:16:58,387 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1008): ClusterId : a6524d16-1e79-42ad-a7f2-328a2e04216d 2024-12-15T06:16:58,387 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:16:58,389 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:16:58,389 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:16:58,391 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:16:58,391 DEBUG [RS:0;00a5f8d66132:37021 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4364a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:16:58,391 DEBUG [RS:0;00a5f8d66132:37021 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39914eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:16:58,391 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:16:58,391 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:16:58,391 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:16:58,392 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,39109,1734243418107 with isa=00a5f8d66132/172.17.0.2:37021, startcode=1734243418154 2024-12-15T06:16:58,392 DEBUG [RS:0;00a5f8d66132:37021 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:16:58,394 INFO [RS-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42473, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:16:58,395 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39109 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,395 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39109 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,396 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5 2024-12-15T06:16:58,396 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:43087 2024-12-15T06:16:58,396 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:16:58,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:16:58,398 DEBUG [RS:0;00a5f8d66132:37021 {}] zookeeper.ZKUtil(111): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,398 WARN [RS:0;00a5f8d66132:37021 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:16:58,398 INFO [RS:0;00a5f8d66132:37021 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:16:58,398 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,399 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,37021,1734243418154] 2024-12-15T06:16:58,401 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:16:58,401 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:16:58,403 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:16:58,403 INFO [RS:0;00a5f8d66132:37021 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:16:58,403 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,403 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:16:58,404 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,404 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,404 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,404 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,404 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,404 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:16:58,405 DEBUG [RS:0;00a5f8d66132:37021 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:16:58,405 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,405 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,405 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,405 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,405 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,37021,1734243418154-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:16:58,420 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:16:58,420 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,37021,1734243418154-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,434 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.Replication(204): 00a5f8d66132,37021,1734243418154 started 2024-12-15T06:16:58,434 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,37021,1734243418154, RpcServer on 00a5f8d66132/172.17.0.2:37021, sessionid=0x100945238d30001 2024-12-15T06:16:58,434 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:16:58,434 DEBUG [RS:0;00a5f8d66132:37021 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,434 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,37021,1734243418154' 2024-12-15T06:16:58,434 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:16:58,435 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:16:58,435 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:16:58,435 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:16:58,435 DEBUG [RS:0;00a5f8d66132:37021 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,435 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,37021,1734243418154' 2024-12-15T06:16:58,435 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:16:58,436 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:16:58,436 DEBUG [RS:0;00a5f8d66132:37021 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:16:58,436 INFO [RS:0;00a5f8d66132:37021 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:16:58,436 INFO [RS:0;00a5f8d66132:37021 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:16:58,471 WARN [00a5f8d66132:39109 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-15T06:16:58,538 INFO [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C37021%2C1734243418154, suffix=, logDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154, archiveDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/oldWALs, maxLogs=32 2024-12-15T06:16:58,539 INFO [RS:0;00a5f8d66132:37021 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37021%2C1734243418154.1734243418538 2024-12-15T06:16:58,544 INFO [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243418538 2024-12-15T06:16:58,544 DEBUG [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44229:44229),(127.0.0.1/127.0.0.1:38289:38289)] 2024-12-15T06:16:58,722 DEBUG [00a5f8d66132:39109 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-15T06:16:58,722 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,723 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,37021,1734243418154, state=OPENING 2024-12-15T06:16:58,724 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-15T06:16:58,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:58,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=00a5f8d66132,37021,1734243418154}] 2024-12-15T06:16:58,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:16:58,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:16:58,853 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:16:58,879 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,879 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-15T06:16:58,881 INFO [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42072, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-15T06:16:58,884 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-15T06:16:58,885 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:16:58,886 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C37021%2C1734243418154.meta, suffix=.meta, logDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154, archiveDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/oldWALs, maxLogs=32 2024-12-15T06:16:58,887 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37021%2C1734243418154.meta.1734243418887.meta 2024-12-15T06:16:58,892 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.meta.1734243418887.meta 2024-12-15T06:16:58,892 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44229:44229),(127.0.0.1/127.0.0.1:38289:38289)] 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-15T06:16:58,893 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-15T06:16:58,893 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-15T06:16:58,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:16:58,895 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:16:58,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:16:58,896 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:16:58,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:16:58,897 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:16:58,897 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:16:58,898 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740 2024-12-15T06:16:58,899 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740 2024-12-15T06:16:58,900 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:16:58,902 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:16:58,903 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776068, jitterRate=-0.013178646564483643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:16:58,903 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:16:58,903 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734243418878 2024-12-15T06:16:58,905 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-15T06:16:58,905 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-15T06:16:58,906 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:58,906 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,37021,1734243418154, state=OPEN 2024-12-15T06:16:58,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:16:58,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:16:58,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:16:58,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:16:58,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-15T06:16:58,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=00a5f8d66132,37021,1734243418154 in 184 msec 2024-12-15T06:16:58,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-15T06:16:58,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 593 msec 2024-12-15T06:16:58,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 627 msec 2024-12-15T06:16:58,916 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734243418916, completionTime=-1 2024-12-15T06:16:58,916 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-15T06:16:58,916 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-15T06:16:58,917 DEBUG [hconnection-0x3c82d692-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:16:58,918 INFO [RS-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:16:58,919 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-15T06:16:58,919 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734243478919 2024-12-15T06:16:58,919 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734243538919 2024-12-15T06:16:58,919 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-15T06:16:58,924 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,39109,1734243418107-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,924 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,39109,1734243418107-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,924 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,39109,1734243418107-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,924 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00a5f8d66132:39109, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,924 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:58,924 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-15T06:16:58,925 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:16:58,925 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-15T06:16:58,926 DEBUG [master/00a5f8d66132:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-15T06:16:58,926 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:16:58,926 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:58,927 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:16:58,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:16:58,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:16:58,936 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 022ec1191931bfa1aecd0bdac1108963, NAME => 'hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5 2024-12-15T06:16:58,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:16:58,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:16:58,942 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:58,942 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 022ec1191931bfa1aecd0bdac1108963, disabling compactions & flushes 2024-12-15T06:16:58,942 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:58,942 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:58,942 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. after waiting 0 ms 2024-12-15T06:16:58,942 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:58,942 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:58,942 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 022ec1191931bfa1aecd0bdac1108963: 2024-12-15T06:16:58,943 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:16:58,944 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734243418943"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243418943"}]},"ts":"1734243418943"} 2024-12-15T06:16:58,945 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:16:58,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:16:58,946 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243418946"}]},"ts":"1734243418946"} 2024-12-15T06:16:58,948 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-15T06:16:58,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=022ec1191931bfa1aecd0bdac1108963, ASSIGN}] 2024-12-15T06:16:58,952 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=022ec1191931bfa1aecd0bdac1108963, ASSIGN 2024-12-15T06:16:58,953 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=022ec1191931bfa1aecd0bdac1108963, ASSIGN; state=OFFLINE, location=00a5f8d66132,37021,1734243418154; forceNewPlan=false, retain=false 2024-12-15T06:16:59,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=022ec1191931bfa1aecd0bdac1108963, regionState=OPENING, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:59,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 022ec1191931bfa1aecd0bdac1108963, server=00a5f8d66132,37021,1734243418154}] 2024-12-15T06:16:59,258 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37021,1734243418154 2024-12-15T06:16:59,261 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:59,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 022ec1191931bfa1aecd0bdac1108963, NAME => 'hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:16:59,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:59,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,263 INFO [StoreOpener-022ec1191931bfa1aecd0bdac1108963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,264 INFO [StoreOpener-022ec1191931bfa1aecd0bdac1108963-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022ec1191931bfa1aecd0bdac1108963 columnFamilyName info 2024-12-15T06:16:59,265 DEBUG [StoreOpener-022ec1191931bfa1aecd0bdac1108963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:59,265 INFO [StoreOpener-022ec1191931bfa1aecd0bdac1108963-1 {}] regionserver.HStore(327): Store=022ec1191931bfa1aecd0bdac1108963/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:16:59,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:16:59,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:16:59,270 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 022ec1191931bfa1aecd0bdac1108963; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=688666, jitterRate=-0.12431655824184418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:16:59,271 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 022ec1191931bfa1aecd0bdac1108963: 2024-12-15T06:16:59,271 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963., pid=6, masterSystemTime=1734243419258 2024-12-15T06:16:59,273 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:59,273 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:16:59,273 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=022ec1191931bfa1aecd0bdac1108963, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:59,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-15T06:16:59,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 022ec1191931bfa1aecd0bdac1108963, server=00a5f8d66132,37021,1734243418154 in 170 msec 2024-12-15T06:16:59,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-15T06:16:59,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=022ec1191931bfa1aecd0bdac1108963, ASSIGN in 326 msec 2024-12-15T06:16:59,279 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:16:59,279 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243419279"}]},"ts":"1734243419279"} 2024-12-15T06:16:59,280 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-15T06:16:59,283 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:16:59,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 358 msec 2024-12-15T06:16:59,327 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-15T06:16:59,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:16:59,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:59,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:16:59,332 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-15T06:16:59,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:16:59,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 10 msec 2024-12-15T06:16:59,344 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-15T06:16:59,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:16:59,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec 2024-12-15T06:16:59,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-15T06:16:59,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.191sec 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,39109,1734243418107-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:16:59,371 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,39109,1734243418107-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-15T06:16:59,373 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-15T06:16:59,373 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-15T06:16:59,373 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,39109,1734243418107-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:16:59,376 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x563a8813 to 127.0.0.1:52277 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f270384 2024-12-15T06:16:59,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6309a231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:16:59,382 DEBUG [hconnection-0x5a304882-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:16:59,383 INFO [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42102, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:16:59,385 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=00a5f8d66132,39109,1734243418107 2024-12-15T06:16:59,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:16:59,388 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-15T06:16:59,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-15T06:16:59,390 INFO [RS-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45230, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-15T06:16:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-15T06:16:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-15T06:16:59,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:16:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-12-15T06:16:59,393 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:16:59,393 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:59,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 9 2024-12-15T06:16:59,394 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:16:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:16:59,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741837_1013 (size=381) 2024-12-15T06:16:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741837_1013 (size=381) 2024-12-15T06:16:59,402 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c66dfc53ca05a54bcf1b32e5070c60b0, NAME => 'TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5 2024-12-15T06:16:59,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741838_1014 (size=64) 2024-12-15T06:16:59,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741838_1014 (size=64) 2024-12-15T06:16:59,409 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:59,409 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing c66dfc53ca05a54bcf1b32e5070c60b0, disabling compactions & flushes 2024-12-15T06:16:59,409 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,409 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,409 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. after waiting 0 ms 2024-12-15T06:16:59,409 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,409 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,409 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:16:59,410 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:16:59,411 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1734243419410"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243419410"}]},"ts":"1734243419410"} 2024-12-15T06:16:59,412 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:16:59,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:16:59,413 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243419413"}]},"ts":"1734243419413"} 2024-12-15T06:16:59,414 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-12-15T06:16:59,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, ASSIGN}] 2024-12-15T06:16:59,419 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, ASSIGN 2024-12-15T06:16:59,419 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, ASSIGN; state=OFFLINE, location=00a5f8d66132,37021,1734243418154; forceNewPlan=false, retain=false 2024-12-15T06:16:59,570 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c66dfc53ca05a54bcf1b32e5070c60b0, regionState=OPENING, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:59,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154}] 2024-12-15T06:16:59,724 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37021,1734243418154 2024-12-15T06:16:59,728 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,728 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => c66dfc53ca05a54bcf1b32e5070c60b0, NAME => 'TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:16:59,728 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,728 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:16:59,729 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,729 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,730 INFO [StoreOpener-c66dfc53ca05a54bcf1b32e5070c60b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,731 INFO [StoreOpener-c66dfc53ca05a54bcf1b32e5070c60b0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c66dfc53ca05a54bcf1b32e5070c60b0 columnFamilyName info 2024-12-15T06:16:59,731 DEBUG [StoreOpener-c66dfc53ca05a54bcf1b32e5070c60b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:16:59,732 INFO [StoreOpener-c66dfc53ca05a54bcf1b32e5070c60b0-1 {}] regionserver.HStore(327): Store=c66dfc53ca05a54bcf1b32e5070c60b0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:16:59,732 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,732 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,734 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:16:59,736 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:16:59,737 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened c66dfc53ca05a54bcf1b32e5070c60b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776563, jitterRate=-0.01255008578300476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:16:59,737 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:16:59,738 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0., pid=11, masterSystemTime=1734243419724 2024-12-15T06:16:59,740 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,740 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:16:59,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c66dfc53ca05a54bcf1b32e5070c60b0, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:16:59,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-15T06:16:59,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154 in 170 msec 2024-12-15T06:16:59,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-15T06:16:59,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, ASSIGN in 326 msec 2024-12-15T06:16:59,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:16:59,746 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243419746"}]},"ts":"1734243419746"} 2024-12-15T06:16:59,748 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-12-15T06:16:59,750 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:16:59,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRolling in 358 msec 2024-12-15T06:16:59,854 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:00,854 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:01,855 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:01,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,978 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,979 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,979 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,979 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,979 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,979 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,982 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,982 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,982 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:01,984 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,489 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:17:02,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,508 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,508 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,511 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,511 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,511 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,513 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:02,855 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:03,856 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:04,402 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-15T06:17:04,402 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-15T06:17:04,403 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-12-15T06:17:04,857 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:04,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-15T06:17:04,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-15T06:17:04,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-15T06:17:05,857 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:06,858 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:07,858 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:08,859 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39109 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-15T06:17:09,396 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling, procId: 9 completed 2024-12-15T06:17:09,398 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRolling 2024-12-15T06:17:09,398 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:09,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:17:09,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/11a00083b24344d0b0ecfb2568cdbee1 is 1080, key is row0001/info:/1734243429401/Put/seqid=0 2024-12-15T06:17:09,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-15T06:17:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42102 deadline: 1734243439434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741839_1015 (size=12509) 2024-12-15T06:17:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741839_1015 (size=12509) 2024-12-15T06:17:09,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/11a00083b24344d0b0ecfb2568cdbee1 2024-12-15T06:17:09,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/11a00083b24344d0b0ecfb2568cdbee1 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/11a00083b24344d0b0ecfb2568cdbee1 2024-12-15T06:17:09,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/11a00083b24344d0b0ecfb2568cdbee1, entries=7, sequenceid=11, filesize=12.2 K 2024-12-15T06:17:09,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for c66dfc53ca05a54bcf1b32e5070c60b0 in 41ms, sequenceid=11, compaction requested=false 2024-12-15T06:17:09,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:09,860 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:10,424 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:17:10,425 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,425 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,426 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,426 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,426 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,426 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,445 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,445 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,445 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,448 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,449 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,449 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,451 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:10,860 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:11,717 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 69ff2480557a57f1cd7d9a62de1d020c, had cached 0 bytes from a total of 23930 2024-12-15T06:17:11,861 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:12,861 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:13,862 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:14,862 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:15,692 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=1, created chunk count=16, reused chunk count=35, reuseRatio=68.63% 2024-12-15T06:17:15,692 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-15T06:17:15,863 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:16,864 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:17,864 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:18,865 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:19,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-15T06:17:19,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/8e77202e54744b7884d0f7d7e863d210 is 1080, key is row0008/info:/1734243429409/Put/seqid=0 2024-12-15T06:17:19,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741840_1016 (size=29761) 2024-12-15T06:17:19,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741840_1016 (size=29761) 2024-12-15T06:17:19,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/8e77202e54744b7884d0f7d7e863d210 2024-12-15T06:17:19,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/8e77202e54744b7884d0f7d7e863d210 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210 2024-12-15T06:17:19,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210, entries=23, sequenceid=37, filesize=29.1 K 2024-12-15T06:17:19,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for c66dfc53ca05a54bcf1b32e5070c60b0 in 25ms, sequenceid=37, compaction requested=false 2024-12-15T06:17:19,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:19,543 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=41.3 K, sizeToCheck=16.0 K 2024-12-15T06:17:19,543 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:19,543 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210 because midkey is the same as first or last row 2024-12-15T06:17:19,865 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:20,866 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:21,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:17:21,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/aeeac9c3b8fc465eae9154ef996ee52d is 1080, key is row0031/info:/1734243439518/Put/seqid=0 2024-12-15T06:17:21,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741841_1017 (size=12509) 2024-12-15T06:17:21,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741841_1017 (size=12509) 2024-12-15T06:17:21,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/aeeac9c3b8fc465eae9154ef996ee52d 2024-12-15T06:17:21,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/aeeac9c3b8fc465eae9154ef996ee52d as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/aeeac9c3b8fc465eae9154ef996ee52d 2024-12-15T06:17:21,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/aeeac9c3b8fc465eae9154ef996ee52d, entries=7, sequenceid=47, filesize=12.2 K 2024-12-15T06:17:21,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=22.07 KB/22596 for c66dfc53ca05a54bcf1b32e5070c60b0 in 24ms, sequenceid=47, compaction requested=true 2024-12-15T06:17:21,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:21,550 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=53.5 K, sizeToCheck=16.0 K 2024-12-15T06:17:21,550 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:21,550 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210 because midkey is the same as first or last row 2024-12-15T06:17:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:21,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c66dfc53ca05a54bcf1b32e5070c60b0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:17:21,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:21,550 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:17:21,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=23.12 KB heapSize=25 KB 2024-12-15T06:17:21,552 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:17:21,552 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): c66dfc53ca05a54bcf1b32e5070c60b0/info is initiating minor compaction (all files) 2024-12-15T06:17:21,552 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of c66dfc53ca05a54bcf1b32e5070c60b0/info in TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:21,552 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/11a00083b24344d0b0ecfb2568cdbee1, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/aeeac9c3b8fc465eae9154ef996ee52d] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp, totalSize=53.5 K 2024-12-15T06:17:21,552 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 11a00083b24344d0b0ecfb2568cdbee1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1734243429401 2024-12-15T06:17:21,553 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8e77202e54744b7884d0f7d7e863d210, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1734243429409 2024-12-15T06:17:21,553 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting aeeac9c3b8fc465eae9154ef996ee52d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1734243439518 2024-12-15T06:17:21,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/991124c520af4c67b037daf9609b366a is 1080, key is row0038/info:/1734243441527/Put/seqid=0 2024-12-15T06:17:21,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741842_1018 (size=28684) 2024-12-15T06:17:21,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741842_1018 (size=28684) 2024-12-15T06:17:21,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/991124c520af4c67b037daf9609b366a 2024-12-15T06:17:21,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/991124c520af4c67b037daf9609b366a as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/991124c520af4c67b037daf9609b366a 2024-12-15T06:17:21,572 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c66dfc53ca05a54bcf1b32e5070c60b0#info#compaction#41 average throughput is 9.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:21,573 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/503dd80f97204178a0dfe52c0e6f3548 is 1080, key is row0001/info:/1734243429401/Put/seqid=0 2024-12-15T06:17:21,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/991124c520af4c67b037daf9609b366a, entries=22, sequenceid=72, filesize=28.0 K 2024-12-15T06:17:21,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=5.25 KB/5380 for c66dfc53ca05a54bcf1b32e5070c60b0 in 24ms, sequenceid=72, compaction requested=false 2024-12-15T06:17:21,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:21,574 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=81.5 K, sizeToCheck=16.0 K 2024-12-15T06:17:21,574 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:21,574 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210 because midkey is the same as first or last row 2024-12-15T06:17:21,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741843_1019 (size=44978) 2024-12-15T06:17:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741843_1019 (size=44978) 2024-12-15T06:17:21,587 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/503dd80f97204178a0dfe52c0e6f3548 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548 2024-12-15T06:17:21,594 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in c66dfc53ca05a54bcf1b32e5070c60b0/info of c66dfc53ca05a54bcf1b32e5070c60b0 into 503dd80f97204178a0dfe52c0e6f3548(size=43.9 K), total size for store is 71.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:21,594 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:21,594 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0., storeName=c66dfc53ca05a54bcf1b32e5070c60b0/info, priority=13, startTime=1734243441550; duration=0sec 2024-12-15T06:17:21,594 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=71.9 K, sizeToCheck=16.0 K 2024-12-15T06:17:21,594 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:21,594 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548 because midkey is the same as first or last row 2024-12-15T06:17:21,594 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:21,594 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c66dfc53ca05a54bcf1b32e5070c60b0:info 2024-12-15T06:17:21,866 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:22,867 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:17:23,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/1e15411e0b5d4c618fa58f68a3f70458 is 1080, key is row0060/info:/1734243441551/Put/seqid=0 2024-12-15T06:17:23,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741844_1020 (size=12509) 2024-12-15T06:17:23,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741844_1020 (size=12509) 2024-12-15T06:17:23,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/1e15411e0b5d4c618fa58f68a3f70458 2024-12-15T06:17:23,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/1e15411e0b5d4c618fa58f68a3f70458 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1e15411e0b5d4c618fa58f68a3f70458 2024-12-15T06:17:23,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1e15411e0b5d4c618fa58f68a3f70458, entries=7, sequenceid=83, filesize=12.2 K 2024-12-15T06:17:23,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for c66dfc53ca05a54bcf1b32e5070c60b0 in 25ms, sequenceid=83, compaction requested=true 2024-12-15T06:17:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=84.2 K, sizeToCheck=16.0 K 2024-12-15T06:17:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548 because midkey is the same as first or last row 2024-12-15T06:17:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c66dfc53ca05a54bcf1b32e5070c60b0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:17:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:23,584 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:17:23,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB 2024-12-15T06:17:23,585 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 86171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:17:23,585 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): c66dfc53ca05a54bcf1b32e5070c60b0/info is initiating minor compaction (all files) 2024-12-15T06:17:23,586 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of c66dfc53ca05a54bcf1b32e5070c60b0/info in TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:23,586 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/991124c520af4c67b037daf9609b366a, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1e15411e0b5d4c618fa58f68a3f70458] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp, totalSize=84.2 K 2024-12-15T06:17:23,586 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 503dd80f97204178a0dfe52c0e6f3548, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1734243429401 2024-12-15T06:17:23,587 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 991124c520af4c67b037daf9609b366a, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1734243441527 2024-12-15T06:17:23,587 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1e15411e0b5d4c618fa58f68a3f70458, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1734243441551 2024-12-15T06:17:23,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/0e7809a140424cbe85a6a20525c10a35 is 1080, key is row0067/info:/1734243443560/Put/seqid=0 2024-12-15T06:17:23,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741845_1021 (size=27607) 2024-12-15T06:17:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-15T06:17:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42102 deadline: 1734243453594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:23,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741845_1021 (size=27607) 2024-12-15T06:17:23,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/0e7809a140424cbe85a6a20525c10a35 2024-12-15T06:17:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/0e7809a140424cbe85a6a20525c10a35 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/0e7809a140424cbe85a6a20525c10a35 2024-12-15T06:17:23,602 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c66dfc53ca05a54bcf1b32e5070c60b0#info#compaction#44 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:23,602 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/1b65be837ec34957aa57cfd4be1a614c is 1080, key is row0001/info:/1734243429401/Put/seqid=0 2024-12-15T06:17:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741846_1022 (size=76455) 2024-12-15T06:17:23,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/0e7809a140424cbe85a6a20525c10a35, entries=21, sequenceid=107, filesize=27.0 K 2024-12-15T06:17:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741846_1022 (size=76455) 2024-12-15T06:17:23,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=8.41 KB/8608 for c66dfc53ca05a54bcf1b32e5070c60b0 in 24ms, sequenceid=107, compaction requested=false 2024-12-15T06:17:23,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:23,608 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=111.1 K, sizeToCheck=16.0 K 2024-12-15T06:17:23,608 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:23,608 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548 because midkey is the same as first or last row 2024-12-15T06:17:23,613 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/1b65be837ec34957aa57cfd4be1a614c as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c 2024-12-15T06:17:23,618 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in c66dfc53ca05a54bcf1b32e5070c60b0/info of c66dfc53ca05a54bcf1b32e5070c60b0 into 1b65be837ec34957aa57cfd4be1a614c(size=74.7 K), total size for store is 101.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:23,618 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:23,618 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0., storeName=c66dfc53ca05a54bcf1b32e5070c60b0/info, priority=13, startTime=1734243443584; duration=0sec 2024-12-15T06:17:23,618 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=101.6 K, sizeToCheck=16.0 K 2024-12-15T06:17:23,619 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-15T06:17:23,619 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:23,619 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:23,619 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c66dfc53ca05a54bcf1b32e5070c60b0:info 2024-12-15T06:17:23,621 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39109 {}] assignment.AssignmentManager(1346): Split request from 00a5f8d66132,37021,1734243418154, parent={ENCODED => c66dfc53ca05a54bcf1b32e5070c60b0, NAME => 'TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-12-15T06:17:23,625 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39109 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:23,629 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39109 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c66dfc53ca05a54bcf1b32e5070c60b0, daughterA=9e2aca8db9b285ac185e87a70150120a, daughterB=da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:23,630 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c66dfc53ca05a54bcf1b32e5070c60b0, daughterA=9e2aca8db9b285ac185e87a70150120a, daughterB=da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:23,630 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c66dfc53ca05a54bcf1b32e5070c60b0, daughterA=9e2aca8db9b285ac185e87a70150120a, daughterB=da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:23,630 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c66dfc53ca05a54bcf1b32e5070c60b0, daughterA=9e2aca8db9b285ac185e87a70150120a, daughterB=da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:23,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, UNASSIGN}] 2024-12-15T06:17:23,637 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, UNASSIGN 2024-12-15T06:17:23,638 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=c66dfc53ca05a54bcf1b32e5070c60b0, regionState=CLOSING, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:23,639 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-12-15T06:17:23,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; CloseRegionProcedure c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154}] 2024-12-15T06:17:23,795 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37021,1734243418154 2024-12-15T06:17:23,796 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(124): Close c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,797 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(138): Unassign region: split region: true: evictCache: true 2024-12-15T06:17:23,797 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1681): Closing c66dfc53ca05a54bcf1b32e5070c60b0, disabling compactions & flushes 2024-12-15T06:17:23,797 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:23,797 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:23,797 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. after waiting 0 ms 2024-12-15T06:17:23,797 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:23,797 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(2837): Flushing c66dfc53ca05a54bcf1b32e5070c60b0 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-12-15T06:17:23,802 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/7b3fc934f1fd422c8ed4fd72cc437135 is 1080, key is row0088/info:/1734243443585/Put/seqid=0 2024-12-15T06:17:23,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741847_1023 (size=13586) 2024-12-15T06:17:23,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741847_1023 (size=13586) 2024-12-15T06:17:23,807 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/7b3fc934f1fd422c8ed4fd72cc437135 2024-12-15T06:17:23,812 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/.tmp/info/7b3fc934f1fd422c8ed4fd72cc437135 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/7b3fc934f1fd422c8ed4fd72cc437135 2024-12-15T06:17:23,817 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/7b3fc934f1fd422c8ed4fd72cc437135, entries=8, sequenceid=119, filesize=13.3 K 2024-12-15T06:17:23,818 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(3040): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for c66dfc53ca05a54bcf1b32e5070c60b0 in 21ms, sequenceid=119, compaction requested=true 2024-12-15T06:17:23,819 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/11a00083b24344d0b0ecfb2568cdbee1, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/aeeac9c3b8fc465eae9154ef996ee52d, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/991124c520af4c67b037daf9609b366a, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1e15411e0b5d4c618fa58f68a3f70458] to archive 2024-12-15T06:17:23,820 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-15T06:17:23,822 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/11a00083b24344d0b0ecfb2568cdbee1 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/11a00083b24344d0b0ecfb2568cdbee1 2024-12-15T06:17:23,822 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/8e77202e54744b7884d0f7d7e863d210 2024-12-15T06:17:23,822 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/503dd80f97204178a0dfe52c0e6f3548 2024-12-15T06:17:23,823 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1e15411e0b5d4c618fa58f68a3f70458 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1e15411e0b5d4c618fa58f68a3f70458 2024-12-15T06:17:23,823 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/991124c520af4c67b037daf9609b366a to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/991124c520af4c67b037daf9609b366a 2024-12-15T06:17:23,823 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/aeeac9c3b8fc465eae9154ef996ee52d to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/aeeac9c3b8fc465eae9154ef996ee52d 2024-12-15T06:17:23,830 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/recovered.edits/122.seqid, newMaxSeqId=122, maxSeqId=1 2024-12-15T06:17:23,831 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. 2024-12-15T06:17:23,831 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1635): Region close journal for c66dfc53ca05a54bcf1b32e5070c60b0: 2024-12-15T06:17:23,832 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(170): Closed c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,833 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=c66dfc53ca05a54bcf1b32e5070c60b0, regionState=CLOSED 2024-12-15T06:17:23,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-12-15T06:17:23,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; CloseRegionProcedure c66dfc53ca05a54bcf1b32e5070c60b0, server=00a5f8d66132,37021,1734243418154 in 194 msec 2024-12-15T06:17:23,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-15T06:17:23,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c66dfc53ca05a54bcf1b32e5070c60b0, UNASSIGN in 201 msec 2024-12-15T06:17:23,858 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:17:23,859 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=12 splitting 3 storefiles, region=c66dfc53ca05a54bcf1b32e5070c60b0, threads=3 2024-12-15T06:17:23,860 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/0e7809a140424cbe85a6a20525c10a35 for region: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,861 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c for region: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,861 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/7b3fc934f1fd422c8ed4fd72cc437135 for region: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,868 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:23,872 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/7b3fc934f1fd422c8ed4fd72cc437135, top=true 2024-12-15T06:17:23,872 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/0e7809a140424cbe85a6a20525c10a35, top=true 2024-12-15T06:17:23,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741848_1024 (size=27) 2024-12-15T06:17:23,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741848_1024 (size=27) 2024-12-15T06:17:23,888 INFO [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35 for child: da9f6afa8670a797d04086949b9e35c0, parent: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,888 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/0e7809a140424cbe85a6a20525c10a35 for region: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,892 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135 for child: da9f6afa8670a797d04086949b9e35c0, parent: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,892 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/7b3fc934f1fd422c8ed4fd72cc437135 for region: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741849_1025 (size=27) 2024-12-15T06:17:23,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741849_1025 (size=27) 2024-12-15T06:17:23,896 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c for region: c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:17:23,897 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=12 split storefiles for region c66dfc53ca05a54bcf1b32e5070c60b0 Daughter A: [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0] storefiles, Daughter B: [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135] storefiles. 2024-12-15T06:17:23,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741850_1026 (size=71) 2024-12-15T06:17:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741850_1026 (size=71) 2024-12-15T06:17:23,907 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:17:23,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741851_1027 (size=71) 2024-12-15T06:17:23,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741851_1027 (size=71) 2024-12-15T06:17:23,922 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:17:23,932 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/recovered.edits/122.seqid, newMaxSeqId=122, maxSeqId=-1 2024-12-15T06:17:23,934 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/recovered.edits/122.seqid, newMaxSeqId=122, maxSeqId=-1 2024-12-15T06:17:23,936 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1734243443935"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1734243443935"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1734243443935"}]},"ts":"1734243443935"} 2024-12-15T06:17:23,936 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1734243443935"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243443935"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1734243443935"}]},"ts":"1734243443935"} 2024-12-15T06:17:23,936 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1734243443935"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243443935"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1734243443935"}]},"ts":"1734243443935"} 2024-12-15T06:17:23,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37021 {}] regionserver.HRegion(8581): Flush requested on 1588230740 2024-12-15T06:17:23,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-15T06:17:23,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=4.75 KB heapSize=8.29 KB 2024-12-15T06:17:23,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9e2aca8db9b285ac185e87a70150120a, ASSIGN}, {pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=da9f6afa8670a797d04086949b9e35c0, ASSIGN}] 2024-12-15T06:17:23,973 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9e2aca8db9b285ac185e87a70150120a, ASSIGN 2024-12-15T06:17:23,973 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=da9f6afa8670a797d04086949b9e35c0, ASSIGN 2024-12-15T06:17:23,974 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9e2aca8db9b285ac185e87a70150120a, ASSIGN; state=SPLITTING_NEW, location=00a5f8d66132,37021,1734243418154; forceNewPlan=false, retain=false 2024-12-15T06:17:23,974 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=da9f6afa8670a797d04086949b9e35c0, ASSIGN; state=SPLITTING_NEW, location=00a5f8d66132,37021,1734243418154; forceNewPlan=false, retain=false 2024-12-15T06:17:23,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/info/c945a5858b994cc6acbbcfc4956525aa is 193, key is TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0./info:regioninfo/1734243443935/Put/seqid=0 2024-12-15T06:17:23,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741852_1028 (size=9423) 2024-12-15T06:17:23,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741852_1028 (size=9423) 2024-12-15T06:17:23,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.54 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/info/c945a5858b994cc6acbbcfc4956525aa 2024-12-15T06:17:24,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/table/db412784b2244f57b712471fd871f875 is 65, key is TestLogRolling-testLogRolling/table:state/1734243419746/Put/seqid=0 2024-12-15T06:17:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741853_1029 (size=5412) 2024-12-15T06:17:24,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741853_1029 (size=5412) 2024-12-15T06:17:24,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=216 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/table/db412784b2244f57b712471fd871f875 2024-12-15T06:17:24,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/info/c945a5858b994cc6acbbcfc4956525aa as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/info/c945a5858b994cc6acbbcfc4956525aa 2024-12-15T06:17:24,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/info/c945a5858b994cc6acbbcfc4956525aa, entries=29, sequenceid=17, filesize=9.2 K 2024-12-15T06:17:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/table/db412784b2244f57b712471fd871f875 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/table/db412784b2244f57b712471fd871f875 2024-12-15T06:17:24,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/table/db412784b2244f57b712471fd871f875, entries=4, sequenceid=17, filesize=5.3 K 2024-12-15T06:17:24,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.75 KB/4869, heapSize ~8.01 KB/8200, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=17, compaction requested=false 2024-12-15T06:17:24,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-15T06:17:24,115 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,115 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,115 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,115 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,115 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,115 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=9e2aca8db9b285ac185e87a70150120a, regionState=OPENING, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:24,124 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=da9f6afa8670a797d04086949b9e35c0, regionState=OPENING, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:24,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=15, state=RUNNABLE; OpenRegionProcedure 9e2aca8db9b285ac185e87a70150120a, server=00a5f8d66132,37021,1734243418154}] 2024-12-15T06:17:24,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=16, state=RUNNABLE; OpenRegionProcedure da9f6afa8670a797d04086949b9e35c0, server=00a5f8d66132,37021,1734243418154}] 2024-12-15T06:17:24,133 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,135 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,138 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,138 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,138 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,140 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,279 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,37021,1734243418154 2024-12-15T06:17:24,282 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:24,282 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7285): Opening region: {ENCODED => da9f6afa8670a797d04086949b9e35c0, NAME => 'TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0.', STARTKEY => 'row0062', ENDKEY => ''} 2024-12-15T06:17:24,283 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,283 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:17:24,283 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7327): checking encryption for da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,283 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7330): checking classloading for da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,284 INFO [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,285 INFO [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da9f6afa8670a797d04086949b9e35c0 columnFamilyName info 2024-12-15T06:17:24,285 DEBUG [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:17:24,297 DEBUG [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0->hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c-top 2024-12-15T06:17:24,301 DEBUG [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35 2024-12-15T06:17:24,305 DEBUG [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135 2024-12-15T06:17:24,305 INFO [StoreOpener-da9f6afa8670a797d04086949b9e35c0-1 {}] regionserver.HStore(327): Store=da9f6afa8670a797d04086949b9e35c0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:17:24,306 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,307 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,309 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1085): writing seq id for da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:24,309 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1102): Opened da9f6afa8670a797d04086949b9e35c0; next sequenceid=123; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=740593, jitterRate=-0.05828768014907837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:17:24,310 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1001): Region open journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:24,311 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., pid=18, masterSystemTime=1734243444279 2024-12-15T06:17:24,311 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(403): Add compact mark for store da9f6afa8670a797d04086949b9e35c0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:17:24,311 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:24,311 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:17:24,312 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:24,312 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): da9f6afa8670a797d04086949b9e35c0/info is initiating minor compaction (all files) 2024-12-15T06:17:24,312 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of da9f6afa8670a797d04086949b9e35c0/info in TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:24,313 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:24,313 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:24,313 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0->hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c-top, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp, totalSize=114.9 K 2024-12-15T06:17:24,313 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:17:24,313 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 9e2aca8db9b285ac185e87a70150120a, NAME => 'TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a.', STARTKEY => '', ENDKEY => 'row0062'} 2024-12-15T06:17:24,313 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,313 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:17:24,313 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0, keycount=33, bloomtype=ROW, size=74.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1734243429401 2024-12-15T06:17:24,313 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,313 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=da9f6afa8670a797d04086949b9e35c0, regionState=OPEN, openSeqNum=123, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:24,313 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,314 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1734243443560 2024-12-15T06:17:24,314 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1734243443585 2024-12-15T06:17:24,315 INFO [StoreOpener-9e2aca8db9b285ac185e87a70150120a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,315 INFO [StoreOpener-9e2aca8db9b285ac185e87a70150120a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e2aca8db9b285ac185e87a70150120a columnFamilyName info 2024-12-15T06:17:24,316 DEBUG [StoreOpener-9e2aca8db9b285ac185e87a70150120a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:17:24,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=16 2024-12-15T06:17:24,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=16, state=SUCCESS; OpenRegionProcedure da9f6afa8670a797d04086949b9e35c0, server=00a5f8d66132,37021,1734243418154 in 188 msec 2024-12-15T06:17:24,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=da9f6afa8670a797d04086949b9e35c0, ASSIGN in 346 msec 2024-12-15T06:17:24,325 DEBUG [StoreOpener-9e2aca8db9b285ac185e87a70150120a-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0->hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c-bottom 2024-12-15T06:17:24,325 INFO [StoreOpener-9e2aca8db9b285ac185e87a70150120a-1 {}] regionserver.HStore(327): Store=9e2aca8db9b285ac185e87a70150120a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:17:24,326 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,327 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,330 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:17:24,330 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1102): Opened 9e2aca8db9b285ac185e87a70150120a; next sequenceid=123; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=733820, jitterRate=-0.06690017879009247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:17:24,330 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 9e2aca8db9b285ac185e87a70150120a: 2024-12-15T06:17:24,331 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a., pid=17, masterSystemTime=1734243444279 2024-12-15T06:17:24,331 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(403): Add compact mark for store 9e2aca8db9b285ac185e87a70150120a:info, priority=-2147483648, current under compaction store size is 2 2024-12-15T06:17:24,331 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:24,331 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-15T06:17:24,332 INFO [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:17:24,332 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HStore(1540): 9e2aca8db9b285ac185e87a70150120a/info is initiating minor compaction (all files) 2024-12-15T06:17:24,332 INFO [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9e2aca8db9b285ac185e87a70150120a/info in TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:17:24,333 INFO [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0->hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c-bottom] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/.tmp, totalSize=74.7 K 2024-12-15T06:17:24,333 DEBUG [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:17:24,333 INFO [RS_OPEN_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:17:24,333 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] compactions.Compactor(224): Compacting 1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0, keycount=33, bloomtype=ROW, size=74.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1734243429401 2024-12-15T06:17:24,334 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=9e2aca8db9b285ac185e87a70150120a, regionState=OPEN, openSeqNum=123, regionLocation=00a5f8d66132,37021,1734243418154 2024-12-15T06:17:24,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=15 2024-12-15T06:17:24,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=15, state=SUCCESS; OpenRegionProcedure 9e2aca8db9b285ac185e87a70150120a, server=00a5f8d66132,37021,1734243418154 in 209 msec 2024-12-15T06:17:24,338 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da9f6afa8670a797d04086949b9e35c0#info#compaction#48 average throughput is 34.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:24,339 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/97174d5c529343018f6af0cb054c1d0f is 1080, key is row0062/info:/1734243441553/Put/seqid=0 2024-12-15T06:17:24,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=12 2024-12-15T06:17:24,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9e2aca8db9b285ac185e87a70150120a, ASSIGN in 365 msec 2024-12-15T06:17:24,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c66dfc53ca05a54bcf1b32e5070c60b0, daughterA=9e2aca8db9b285ac185e87a70150120a, daughterB=da9f6afa8670a797d04086949b9e35c0 in 715 msec 2024-12-15T06:17:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741854_1030 (size=41907) 2024-12-15T06:17:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741854_1030 (size=41907) 2024-12-15T06:17:24,355 INFO [RS:0;00a5f8d66132:37021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e2aca8db9b285ac185e87a70150120a#info#compaction#49 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:24,355 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/.tmp/info/0ec96cce2f854a7e8cac95224c35a716 is 1080, key is row0001/info:/1734243429401/Put/seqid=0 2024-12-15T06:17:24,357 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/97174d5c529343018f6af0cb054c1d0f as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/97174d5c529343018f6af0cb054c1d0f 2024-12-15T06:17:24,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741855_1031 (size=70862) 2024-12-15T06:17:24,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741855_1031 (size=70862) 2024-12-15T06:17:24,364 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in da9f6afa8670a797d04086949b9e35c0/info of da9f6afa8670a797d04086949b9e35c0 into 97174d5c529343018f6af0cb054c1d0f(size=40.9 K), total size for store is 40.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:24,364 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:24,364 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., storeName=da9f6afa8670a797d04086949b9e35c0/info, priority=13, startTime=1734243444311; duration=0sec 2024-12-15T06:17:24,364 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:24,364 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da9f6afa8670a797d04086949b9e35c0:info 2024-12-15T06:17:24,367 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/.tmp/info/0ec96cce2f854a7e8cac95224c35a716 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/0ec96cce2f854a7e8cac95224c35a716 2024-12-15T06:17:24,372 INFO [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 1 (all) file(s) in 9e2aca8db9b285ac185e87a70150120a/info of 9e2aca8db9b285ac185e87a70150120a into 0ec96cce2f854a7e8cac95224c35a716(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:24,372 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9e2aca8db9b285ac185e87a70150120a: 2024-12-15T06:17:24,372 INFO [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a., storeName=9e2aca8db9b285ac185e87a70150120a/info, priority=15, startTime=1734243444331; duration=0sec 2024-12-15T06:17:24,372 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:24,373 DEBUG [RS:0;00a5f8d66132:37021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e2aca8db9b285ac185e87a70150120a:info 2024-12-15T06:17:24,649 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:17:24,650 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,650 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,650 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,650 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,650 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,651 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,669 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,669 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,674 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,676 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:24,868 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:25,869 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:26,869 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:27,870 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:28,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:17:28,870 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:29,391 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:17:29,393 INFO [RS-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54208, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:17:29,785 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-15T06:17:29,786 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,804 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,804 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,804 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,804 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,805 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,805 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,808 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,808 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,808 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,810 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-15T06:17:29,871 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:30,871 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:31,872 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:32,873 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42102 deadline: 1734243463679, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1734243419391.c66dfc53ca05a54bcf1b32e5070c60b0. is not online on 00a5f8d66132,37021,1734243418154 2024-12-15T06:17:33,873 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:34,874 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:35,874 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:36,875 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:37,875 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:38,876 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:39,877 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:40,167 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-15T06:17:40,167 INFO [master/00a5f8d66132:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-15T06:17:40,877 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:41,878 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:42,878 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:43,879 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:43,893 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835 2024-12-15T06:17:44,879 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:45,880 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:46,880 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:47,881 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:48,882 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:49,882 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:50,883 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:51,883 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:52,884 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:53,885 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:54,885 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:55,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:17:55,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/7175813ab1914bcf8cae42d33e41c288 is 1080, key is row0096/info:/1734243473750/Put/seqid=0 2024-12-15T06:17:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741856_1032 (size=12514) 2024-12-15T06:17:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741856_1032 (size=12514) 2024-12-15T06:17:55,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/7175813ab1914bcf8cae42d33e41c288 2024-12-15T06:17:55,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/7175813ab1914bcf8cae42d33e41c288 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/7175813ab1914bcf8cae42d33e41c288 2024-12-15T06:17:55,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/7175813ab1914bcf8cae42d33e41c288, entries=7, sequenceid=133, filesize=12.2 K 2024-12-15T06:17:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=24.17 KB/24748 for da9f6afa8670a797d04086949b9e35c0 in 26ms, sequenceid=133, compaction requested=false 2024-12-15T06:17:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:55,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=25.22 KB heapSize=27.25 KB 2024-12-15T06:17:55,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/e02a9de5d5e2489dba420443be289406 is 1080, key is row0103/info:/1734243475759/Put/seqid=0 2024-12-15T06:17:55,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741857_1033 (size=30862) 2024-12-15T06:17:55,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741857_1033 (size=30862) 2024-12-15T06:17:55,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25.22 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/e02a9de5d5e2489dba420443be289406 2024-12-15T06:17:55,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/e02a9de5d5e2489dba420443be289406 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e02a9de5d5e2489dba420443be289406 2024-12-15T06:17:55,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e02a9de5d5e2489dba420443be289406, entries=24, sequenceid=160, filesize=30.1 K 2024-12-15T06:17:55,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.22 KB/25824, heapSize ~27.23 KB/27888, currentSize=2.10 KB/2152 for da9f6afa8670a797d04086949b9e35c0 in 20ms, sequenceid=160, compaction requested=true 2024-12-15T06:17:55,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:55,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da9f6afa8670a797d04086949b9e35c0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:17:55,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:55,805 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:17:55,806 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:17:55,806 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): da9f6afa8670a797d04086949b9e35c0/info is initiating minor compaction (all files) 2024-12-15T06:17:55,806 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of da9f6afa8670a797d04086949b9e35c0/info in TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:55,806 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/97174d5c529343018f6af0cb054c1d0f, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/7175813ab1914bcf8cae42d33e41c288, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e02a9de5d5e2489dba420443be289406] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp, totalSize=83.3 K 2024-12-15T06:17:55,807 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 97174d5c529343018f6af0cb054c1d0f, keycount=34, bloomtype=ROW, size=40.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1734243441553 2024-12-15T06:17:55,807 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7175813ab1914bcf8cae42d33e41c288, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1734243473750 2024-12-15T06:17:55,807 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting e02a9de5d5e2489dba420443be289406, keycount=24, bloomtype=ROW, size=30.1 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1734243475759 2024-12-15T06:17:55,821 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da9f6afa8670a797d04086949b9e35c0#info#compaction#52 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:55,821 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/ec74a92e80114002b5e3aa726f30e6ef is 1080, key is row0062/info:/1734243441553/Put/seqid=0 2024-12-15T06:17:55,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741858_1034 (size=75566) 2024-12-15T06:17:55,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741858_1034 (size=75566) 2024-12-15T06:17:55,831 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/ec74a92e80114002b5e3aa726f30e6ef as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ec74a92e80114002b5e3aa726f30e6ef 2024-12-15T06:17:55,837 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in da9f6afa8670a797d04086949b9e35c0/info of da9f6afa8670a797d04086949b9e35c0 into ec74a92e80114002b5e3aa726f30e6ef(size=73.8 K), total size for store is 73.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:55,837 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:55,837 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., storeName=da9f6afa8670a797d04086949b9e35c0/info, priority=13, startTime=1734243475805; duration=0sec 2024-12-15T06:17:55,837 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:55,837 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da9f6afa8670a797d04086949b9e35c0:info 2024-12-15T06:17:55,886 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:56,717 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 69ff2480557a57f1cd7d9a62de1d020c, had cached 0 bytes from a total of 23930 2024-12-15T06:17:56,886 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:57,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:17:57,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/c1c90f45dc1d4e718ba8b07bac90fc9c is 1080, key is row0127/info:/1734243475785/Put/seqid=0 2024-12-15T06:17:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741859_1035 (size=12516) 2024-12-15T06:17:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741859_1035 (size=12516) 2024-12-15T06:17:57,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/c1c90f45dc1d4e718ba8b07bac90fc9c 2024-12-15T06:17:57,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/c1c90f45dc1d4e718ba8b07bac90fc9c as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/c1c90f45dc1d4e718ba8b07bac90fc9c 2024-12-15T06:17:57,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/c1c90f45dc1d4e718ba8b07bac90fc9c, entries=7, sequenceid=171, filesize=12.2 K 2024-12-15T06:17:57,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for da9f6afa8670a797d04086949b9e35c0 in 47ms, sequenceid=171, compaction requested=false 2024-12-15T06:17:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:57,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-15T06:17:57,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/3904e5f3c4cc4720b4d4f4028b446cfd is 1080, key is row0134/info:/1734243477793/Put/seqid=0 2024-12-15T06:17:57,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741860_1036 (size=29784) 2024-12-15T06:17:57,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741860_1036 (size=29784) 2024-12-15T06:17:57,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/3904e5f3c4cc4720b4d4f4028b446cfd 2024-12-15T06:17:57,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/3904e5f3c4cc4720b4d4f4028b446cfd as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/3904e5f3c4cc4720b4d4f4028b446cfd 2024-12-15T06:17:57,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/3904e5f3c4cc4720b4d4f4028b446cfd, entries=23, sequenceid=197, filesize=29.1 K 2024-12-15T06:17:57,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for da9f6afa8670a797d04086949b9e35c0 in 24ms, sequenceid=197, compaction requested=true 2024-12-15T06:17:57,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da9f6afa8670a797d04086949b9e35c0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:17:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:57,866 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:17:57,867 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 117866 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:17:57,867 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): da9f6afa8670a797d04086949b9e35c0/info is initiating minor compaction (all files) 2024-12-15T06:17:57,867 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of da9f6afa8670a797d04086949b9e35c0/info in TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:57,867 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ec74a92e80114002b5e3aa726f30e6ef, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/c1c90f45dc1d4e718ba8b07bac90fc9c, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/3904e5f3c4cc4720b4d4f4028b446cfd] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp, totalSize=115.1 K 2024-12-15T06:17:57,867 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting ec74a92e80114002b5e3aa726f30e6ef, keycount=65, bloomtype=ROW, size=73.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1734243441553 2024-12-15T06:17:57,868 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting c1c90f45dc1d4e718ba8b07bac90fc9c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1734243475785 2024-12-15T06:17:57,868 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3904e5f3c4cc4720b4d4f4028b446cfd, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1734243477793 2024-12-15T06:17:57,879 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da9f6afa8670a797d04086949b9e35c0#info#compaction#55 average throughput is 48.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:57,879 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/bbbc3c6ecb64467b9a5008d16d94ab2f is 1080, key is row0062/info:/1734243441553/Put/seqid=0 2024-12-15T06:17:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741861_1037 (size=108016) 2024-12-15T06:17:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741861_1037 (size=108016) 2024-12-15T06:17:57,887 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:57,889 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/bbbc3c6ecb64467b9a5008d16d94ab2f as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bbbc3c6ecb64467b9a5008d16d94ab2f 2024-12-15T06:17:57,895 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in da9f6afa8670a797d04086949b9e35c0/info of da9f6afa8670a797d04086949b9e35c0 into bbbc3c6ecb64467b9a5008d16d94ab2f(size=105.5 K), total size for store is 105.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:57,895 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:57,895 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., storeName=da9f6afa8670a797d04086949b9e35c0/info, priority=13, startTime=1734243477866; duration=0sec 2024-12-15T06:17:57,895 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:57,895 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da9f6afa8670a797d04086949b9e35c0:info 2024-12-15T06:17:58,091 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-15T06:17:58,888 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:59,008 DEBUG [master/00a5f8d66132:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-15T06:17:59,008 DEBUG [master/00a5f8d66132:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 022ec1191931bfa1aecd0bdac1108963 changed from -1.0 to 0.0, refreshing cache 2024-12-15T06:17:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:59,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:17:59,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/bc2e2df67cc64d7188cb56ce1d6632cf is 1080, key is row0157/info:/1734243477842/Put/seqid=0 2024-12-15T06:17:59,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741862_1038 (size=12516) 2024-12-15T06:17:59,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741862_1038 (size=12516) 2024-12-15T06:17:59,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/bc2e2df67cc64d7188cb56ce1d6632cf 2024-12-15T06:17:59,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/bc2e2df67cc64d7188cb56ce1d6632cf as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bc2e2df67cc64d7188cb56ce1d6632cf 2024-12-15T06:17:59,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bc2e2df67cc64d7188cb56ce1d6632cf, entries=7, sequenceid=208, filesize=12.2 K 2024-12-15T06:17:59,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for da9f6afa8670a797d04086949b9e35c0 in 23ms, sequenceid=208, compaction requested=false 2024-12-15T06:17:59,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:17:59,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=23.12 KB heapSize=25 KB 2024-12-15T06:17:59,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/34960061cb5a4cca94f5bdd537afdb57 is 1080, key is row0164/info:/1734243479851/Put/seqid=0 2024-12-15T06:17:59,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741863_1039 (size=28706) 2024-12-15T06:17:59,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741863_1039 (size=28706) 2024-12-15T06:17:59,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/34960061cb5a4cca94f5bdd537afdb57 2024-12-15T06:17:59,888 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:17:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/34960061cb5a4cca94f5bdd537afdb57 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/34960061cb5a4cca94f5bdd537afdb57 2024-12-15T06:17:59,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/34960061cb5a4cca94f5bdd537afdb57, entries=22, sequenceid=233, filesize=28.0 K 2024-12-15T06:17:59,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=7.36 KB/7532 for da9f6afa8670a797d04086949b9e35c0 in 22ms, sequenceid=233, compaction requested=true 2024-12-15T06:17:59,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:59,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da9f6afa8670a797d04086949b9e35c0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:17:59,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:59,897 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:17:59,898 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 149238 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:17:59,898 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): da9f6afa8670a797d04086949b9e35c0/info is initiating minor compaction (all files) 2024-12-15T06:17:59,898 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of da9f6afa8670a797d04086949b9e35c0/info in TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:17:59,898 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bbbc3c6ecb64467b9a5008d16d94ab2f, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bc2e2df67cc64d7188cb56ce1d6632cf, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/34960061cb5a4cca94f5bdd537afdb57] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp, totalSize=145.7 K 2024-12-15T06:17:59,898 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting bbbc3c6ecb64467b9a5008d16d94ab2f, keycount=95, bloomtype=ROW, size=105.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1734243441553 2024-12-15T06:17:59,898 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting bc2e2df67cc64d7188cb56ce1d6632cf, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1734243477842 2024-12-15T06:17:59,899 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 34960061cb5a4cca94f5bdd537afdb57, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1734243479851 2024-12-15T06:17:59,909 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da9f6afa8670a797d04086949b9e35c0#info#compaction#58 average throughput is 42.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:17:59,909 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/786cfc6f95b24be29e1d8ea20a5fe085 is 1080, key is row0062/info:/1734243441553/Put/seqid=0 2024-12-15T06:17:59,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741864_1040 (size=139585) 2024-12-15T06:17:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741864_1040 (size=139585) 2024-12-15T06:17:59,918 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/786cfc6f95b24be29e1d8ea20a5fe085 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/786cfc6f95b24be29e1d8ea20a5fe085 2024-12-15T06:17:59,925 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in da9f6afa8670a797d04086949b9e35c0/info of da9f6afa8670a797d04086949b9e35c0 into 786cfc6f95b24be29e1d8ea20a5fe085(size=136.3 K), total size for store is 136.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:17:59,925 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:17:59,925 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., storeName=da9f6afa8670a797d04086949b9e35c0/info, priority=13, startTime=1734243479897; duration=0sec 2024-12-15T06:17:59,925 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:17:59,925 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da9f6afa8670a797d04086949b9e35c0:info 2024-12-15T06:18:00,889 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:01,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-12-15T06:18:01,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/d92f0acf43434b10bcb2ee30dc7c1578 is 1080, key is row0186/info:/1734243479874/Put/seqid=0 2024-12-15T06:18:01,890 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:01,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741865_1041 (size=13594) 2024-12-15T06:18:01,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741865_1041 (size=13594) 2024-12-15T06:18:01,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/d92f0acf43434b10bcb2ee30dc7c1578 2024-12-15T06:18:01,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/d92f0acf43434b10bcb2ee30dc7c1578 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/d92f0acf43434b10bcb2ee30dc7c1578 2024-12-15T06:18:01,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/d92f0acf43434b10bcb2ee30dc7c1578, entries=8, sequenceid=245, filesize=13.3 K 2024-12-15T06:18:01,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=22.07 KB/22596 for da9f6afa8670a797d04086949b9e35c0 in 23ms, sequenceid=245, compaction requested=false 2024-12-15T06:18:01,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:01,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-15T06:18:01,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/337a4821eedb4e6dbcf12853a63040a8 is 1080, key is row0194/info:/1734243481884/Put/seqid=0 2024-12-15T06:18:01,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=da9f6afa8670a797d04086949b9e35c0, server=00a5f8d66132,37021,1734243418154 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-15T06:18:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:42102 deadline: 1734243491913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=da9f6afa8670a797d04086949b9e35c0, server=00a5f8d66132,37021,1734243418154 2024-12-15T06:18:01,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741866_1042 (size=29798) 2024-12-15T06:18:01,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741866_1042 (size=29798) 2024-12-15T06:18:01,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/337a4821eedb4e6dbcf12853a63040a8 2024-12-15T06:18:01,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/337a4821eedb4e6dbcf12853a63040a8 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/337a4821eedb4e6dbcf12853a63040a8 2024-12-15T06:18:01,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/337a4821eedb4e6dbcf12853a63040a8, entries=23, sequenceid=271, filesize=29.1 K 2024-12-15T06:18:01,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for da9f6afa8670a797d04086949b9e35c0 in 20ms, sequenceid=271, compaction requested=true 2024-12-15T06:18:01,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:01,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da9f6afa8670a797d04086949b9e35c0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:18:01,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:18:01,927 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:18:01,928 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 182977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:18:01,928 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): da9f6afa8670a797d04086949b9e35c0/info is initiating minor compaction (all files) 2024-12-15T06:18:01,928 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of da9f6afa8670a797d04086949b9e35c0/info in TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:01,928 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/786cfc6f95b24be29e1d8ea20a5fe085, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/d92f0acf43434b10bcb2ee30dc7c1578, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/337a4821eedb4e6dbcf12853a63040a8] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp, totalSize=178.7 K 2024-12-15T06:18:01,929 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 786cfc6f95b24be29e1d8ea20a5fe085, keycount=124, bloomtype=ROW, size=136.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1734243441553 2024-12-15T06:18:01,929 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting d92f0acf43434b10bcb2ee30dc7c1578, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1734243479874 2024-12-15T06:18:01,929 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 337a4821eedb4e6dbcf12853a63040a8, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1734243481884 2024-12-15T06:18:01,942 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da9f6afa8670a797d04086949b9e35c0#info#compaction#61 average throughput is 53.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:18:01,942 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/0608098ce44f4055b7697c2a991074a8 is 1080, key is row0062/info:/1734243441553/Put/seqid=0 2024-12-15T06:18:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741867_1043 (size=173127) 2024-12-15T06:18:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741867_1043 (size=173127) 2024-12-15T06:18:01,951 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/0608098ce44f4055b7697c2a991074a8 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0608098ce44f4055b7697c2a991074a8 2024-12-15T06:18:01,956 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in da9f6afa8670a797d04086949b9e35c0/info of da9f6afa8670a797d04086949b9e35c0 into 0608098ce44f4055b7697c2a991074a8(size=169.1 K), total size for store is 169.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:18:01,956 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:01,956 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., storeName=da9f6afa8670a797d04086949b9e35c0/info, priority=13, startTime=1734243481927; duration=0sec 2024-12-15T06:18:01,956 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:18:01,956 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da9f6afa8670a797d04086949b9e35c0:info 2024-12-15T06:18:02,890 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:03,891 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:04,891 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:05,892 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:06,892 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:07,893 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:08,894 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:09,283 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region da9f6afa8670a797d04086949b9e35c0, had cached 0 bytes from a total of 173127 2024-12-15T06:18:09,314 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 9e2aca8db9b285ac185e87a70150120a, had cached 0 bytes from a total of 70862 2024-12-15T06:18:09,894 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:10,895 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:11,895 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:11,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:18:11,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/ce0018d1299d474b97b20bac27ef097b is 1080, key is row0217/info:/1734243481908/Put/seqid=0 2024-12-15T06:18:11,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741868_1044 (size=12523) 2024-12-15T06:18:11,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741868_1044 (size=12523) 2024-12-15T06:18:11,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/ce0018d1299d474b97b20bac27ef097b 2024-12-15T06:18:11,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/ce0018d1299d474b97b20bac27ef097b as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ce0018d1299d474b97b20bac27ef097b 2024-12-15T06:18:11,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ce0018d1299d474b97b20bac27ef097b, entries=7, sequenceid=282, filesize=12.2 K 2024-12-15T06:18:11,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=1.05 KB/1076 for da9f6afa8670a797d04086949b9e35c0 in 23ms, sequenceid=282, compaction requested=false 2024-12-15T06:18:11,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:12,896 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:13,897 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:13,897 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta after 196120ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor238.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-15T06:18:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:13,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-15T06:18:13,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/0de993ca371b49ddbfdf36dc2fa07461 is 1080, key is row0224/info:/1734243491974/Put/seqid=0 2024-12-15T06:18:13,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741869_1045 (size=12523) 2024-12-15T06:18:13,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741869_1045 (size=12523) 2024-12-15T06:18:13,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/0de993ca371b49ddbfdf36dc2fa07461 2024-12-15T06:18:13,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/0de993ca371b49ddbfdf36dc2fa07461 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0de993ca371b49ddbfdf36dc2fa07461 2024-12-15T06:18:14,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0de993ca371b49ddbfdf36dc2fa07461, entries=7, sequenceid=292, filesize=12.2 K 2024-12-15T06:18:14,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for da9f6afa8670a797d04086949b9e35c0 in 22ms, sequenceid=292, compaction requested=true 2024-12-15T06:18:14,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:14,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da9f6afa8670a797d04086949b9e35c0:info, priority=-2147483648, current under compaction store size is 1 2024-12-15T06:18:14,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:18:14,003 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-15T06:18:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37021 {}] regionserver.HRegion(8581): Flush requested on da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:14,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB 2024-12-15T06:18:14,004 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 198173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-15T06:18:14,005 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1540): da9f6afa8670a797d04086949b9e35c0/info is initiating minor compaction (all files) 2024-12-15T06:18:14,005 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of da9f6afa8670a797d04086949b9e35c0/info in TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:14,005 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0608098ce44f4055b7697c2a991074a8, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ce0018d1299d474b97b20bac27ef097b, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0de993ca371b49ddbfdf36dc2fa07461] into tmpdir=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp, totalSize=193.5 K 2024-12-15T06:18:14,005 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0608098ce44f4055b7697c2a991074a8, keycount=155, bloomtype=ROW, size=169.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1734243441553 2024-12-15T06:18:14,006 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting ce0018d1299d474b97b20bac27ef097b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1734243481908 2024-12-15T06:18:14,006 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0de993ca371b49ddbfdf36dc2fa07461, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1734243491974 2024-12-15T06:18:14,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/707220fba6764ddf9bc7bea7ea8720a8 is 1080, key is row0231/info:/1734243493981/Put/seqid=0 2024-12-15T06:18:14,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741870_1046 (size=27649) 2024-12-15T06:18:14,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741870_1046 (size=27649) 2024-12-15T06:18:14,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/707220fba6764ddf9bc7bea7ea8720a8 2024-12-15T06:18:14,023 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da9f6afa8670a797d04086949b9e35c0#info#compaction#65 average throughput is 57.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-15T06:18:14,024 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/e597cf65c6b043f3a446de7847e5a8df is 1080, key is row0062/info:/1734243441553/Put/seqid=0 2024-12-15T06:18:14,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/707220fba6764ddf9bc7bea7ea8720a8 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/707220fba6764ddf9bc7bea7ea8720a8 2024-12-15T06:18:14,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/707220fba6764ddf9bc7bea7ea8720a8, entries=21, sequenceid=316, filesize=27.0 K 2024-12-15T06:18:14,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=5.25 KB/5380 for da9f6afa8670a797d04086949b9e35c0 in 27ms, sequenceid=316, compaction requested=false 2024-12-15T06:18:14,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741871_1047 (size=188339) 2024-12-15T06:18:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741871_1047 (size=188339) 2024-12-15T06:18:14,038 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/e597cf65c6b043f3a446de7847e5a8df as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e597cf65c6b043f3a446de7847e5a8df 2024-12-15T06:18:14,044 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in da9f6afa8670a797d04086949b9e35c0/info of da9f6afa8670a797d04086949b9e35c0 into e597cf65c6b043f3a446de7847e5a8df(size=183.9 K), total size for store is 210.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-15T06:18:14,044 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:14,044 INFO [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0., storeName=da9f6afa8670a797d04086949b9e35c0/info, priority=13, startTime=1734243494003; duration=0sec 2024-12-15T06:18:14,044 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-15T06:18:14,044 DEBUG [RS:0;00a5f8d66132:37021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da9f6afa8670a797d04086949b9e35c0:info 2024-12-15T06:18:14,897 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:15,898 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:16,009 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-12-15T06:18:16,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37021%2C1734243418154.1734243496009 2024-12-15T06:18:16,017 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243418538 with entries=308, filesize=306.54 KB; new WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243496009 2024-12-15T06:18:16,017 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44229:44229),(127.0.0.1/127.0.0.1:38289:38289)] 2024-12-15T06:18:16,017 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243418538 is not closed yet, will try archiving it next time 2024-12-15T06:18:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741833_1009 (size=313906) 2024-12-15T06:18:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741833_1009 (size=313906) 2024-12-15T06:18:16,021 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 9e2aca8db9b285ac185e87a70150120a: 2024-12-15T06:18:16,021 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.21 KB heapSize=4.13 KB 2024-12-15T06:18:16,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/info/17f7df7380774711b0f2274923b42711 is 193, key is TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0./info:regioninfo/1734243444313/Put/seqid=0 2024-12-15T06:18:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741873_1049 (size=7803) 2024-12-15T06:18:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741873_1049 (size=7803) 2024-12-15T06:18:16,030 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.21 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/info/17f7df7380774711b0f2274923b42711 2024-12-15T06:18:16,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/.tmp/info/17f7df7380774711b0f2274923b42711 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/info/17f7df7380774711b0f2274923b42711 2024-12-15T06:18:16,040 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/info/17f7df7380774711b0f2274923b42711, entries=16, sequenceid=24, filesize=7.6 K 2024-12-15T06:18:16,040 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~2.21 KB/2260, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 19ms, sequenceid=24, compaction requested=false 2024-12-15T06:18:16,041 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-15T06:18:16,041 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 022ec1191931bfa1aecd0bdac1108963 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-15T06:18:16,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/.tmp/info/70e2360e807c475d8ebf619af47d78b7 is 45, key is default/info:d/1734243419336/Put/seqid=0 2024-12-15T06:18:16,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741874_1050 (size=5037) 2024-12-15T06:18:16,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741874_1050 (size=5037) 2024-12-15T06:18:16,061 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/.tmp/info/70e2360e807c475d8ebf619af47d78b7 2024-12-15T06:18:16,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/.tmp/info/70e2360e807c475d8ebf619af47d78b7 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/info/70e2360e807c475d8ebf619af47d78b7 2024-12-15T06:18:16,069 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/info/70e2360e807c475d8ebf619af47d78b7, entries=2, sequenceid=6, filesize=4.9 K 2024-12-15T06:18:16,070 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 022ec1191931bfa1aecd0bdac1108963 in 29ms, sequenceid=6, compaction requested=false 2024-12-15T06:18:16,070 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 022ec1191931bfa1aecd0bdac1108963: 2024-12-15T06:18:16,070 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing da9f6afa8670a797d04086949b9e35c0 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB 2024-12-15T06:18:16,073 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/b4ee4aafb76b4a1e9bfe14b2ccee3969 is 1080, key is row0252/info:/1734243494004/Put/seqid=0 2024-12-15T06:18:16,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741875_1051 (size=10357) 2024-12-15T06:18:16,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741875_1051 (size=10357) 2024-12-15T06:18:16,078 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/b4ee4aafb76b4a1e9bfe14b2ccee3969 2024-12-15T06:18:16,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/.tmp/info/b4ee4aafb76b4a1e9bfe14b2ccee3969 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/b4ee4aafb76b4a1e9bfe14b2ccee3969 2024-12-15T06:18:16,086 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/b4ee4aafb76b4a1e9bfe14b2ccee3969, entries=5, sequenceid=325, filesize=10.1 K 2024-12-15T06:18:16,087 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for da9f6afa8670a797d04086949b9e35c0 in 17ms, sequenceid=325, compaction requested=true 2024-12-15T06:18:16,087 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:16,087 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C37021%2C1734243418154.1734243496087 2024-12-15T06:18:16,093 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243496009 with entries=4, filesize=1.22 KB; new WAL /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243496087 2024-12-15T06:18:16,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44229:44229),(127.0.0.1/127.0.0.1:38289:38289)] 2024-12-15T06:18:16,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243496009 is not closed yet, will try archiving it next time 2024-12-15T06:18:16,094 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243418538 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/oldWALs/00a5f8d66132%2C37021%2C1734243418154.1734243418538 2024-12-15T06:18:16,094 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-15T06:18:16,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741872_1048 (size=1255) 2024-12-15T06:18:16,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741872_1048 (size=1255) 2024-12-15T06:18:16,095 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154/00a5f8d66132%2C37021%2C1734243418154.1734243496009 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/oldWALs/00a5f8d66132%2C37021%2C1734243418154.1734243496009 2024-12-15T06:18:16,194 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-15T06:18:16,194 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-15T06:18:16,194 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x563a8813 to 127.0.0.1:52277 2024-12-15T06:18:16,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:16,195 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-15T06:18:16,195 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=455848652, stopped=false 2024-12-15T06:18:16,195 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=00a5f8d66132,39109,1734243418107 2024-12-15T06:18:16,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:18:16,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:18:16,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:16,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-15T06:18:16,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:16,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:16,197 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,37021,1734243418154' ***** 2024-12-15T06:18:16,197 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:18:16,198 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:18:16,198 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:18:16,198 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(3579): Received CLOSE for 9e2aca8db9b285ac185e87a70150120a 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(3579): Received CLOSE for 022ec1191931bfa1aecd0bdac1108963 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(3579): Received CLOSE for da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:16,198 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9e2aca8db9b285ac185e87a70150120a, disabling compactions & flushes 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,37021,1734243418154 2024-12-15T06:18:16,198 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:18:16,198 DEBUG [RS:0;00a5f8d66132:37021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:16,198 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:18:16,198 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. after waiting 0 ms 2024-12-15T06:18:16,198 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:18:16,198 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:18:16,199 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:18:16,199 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:18:16,199 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-12-15T06:18:16,199 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1603): Online Regions={9e2aca8db9b285ac185e87a70150120a=TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a., 1588230740=hbase:meta,,1.1588230740, 022ec1191931bfa1aecd0bdac1108963=hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963., da9f6afa8670a797d04086949b9e35c0=TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0.} 2024-12-15T06:18:16,199 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0->hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c-bottom] to archive 2024-12-15T06:18:16,199 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:18:16,199 DEBUG [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1629): Waiting on 022ec1191931bfa1aecd0bdac1108963, 1588230740, 9e2aca8db9b285ac185e87a70150120a, da9f6afa8670a797d04086949b9e35c0 2024-12-15T06:18:16,199 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:18:16,199 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:18:16,199 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:18:16,199 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:18:16,200 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-15T06:18:16,202 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:18:16,203 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/meta/1588230740/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2024-12-15T06:18:16,204 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-15T06:18:16,204 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:18:16,204 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:18:16,204 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-15T06:18:16,205 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/9e2aca8db9b285ac185e87a70150120a/recovered.edits/127.seqid, newMaxSeqId=127, maxSeqId=122 2024-12-15T06:18:16,206 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:18:16,206 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9e2aca8db9b285ac185e87a70150120a: 2024-12-15T06:18:16,206 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1734243443625.9e2aca8db9b285ac185e87a70150120a. 2024-12-15T06:18:16,206 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 022ec1191931bfa1aecd0bdac1108963, disabling compactions & flushes 2024-12-15T06:18:16,206 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:18:16,206 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:18:16,206 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. after waiting 0 ms 2024-12-15T06:18:16,206 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:18:16,209 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/hbase/namespace/022ec1191931bfa1aecd0bdac1108963/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-15T06:18:16,210 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:18:16,210 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 022ec1191931bfa1aecd0bdac1108963: 2024-12-15T06:18:16,210 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734243418924.022ec1191931bfa1aecd0bdac1108963. 2024-12-15T06:18:16,210 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing da9f6afa8670a797d04086949b9e35c0, disabling compactions & flushes 2024-12-15T06:18:16,210 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:16,210 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:16,210 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. after waiting 0 ms 2024-12-15T06:18:16,210 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:16,211 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0->hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/c66dfc53ca05a54bcf1b32e5070c60b0/info/1b65be837ec34957aa57cfd4be1a614c-top, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/97174d5c529343018f6af0cb054c1d0f, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/7175813ab1914bcf8cae42d33e41c288, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ec74a92e80114002b5e3aa726f30e6ef, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e02a9de5d5e2489dba420443be289406, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/c1c90f45dc1d4e718ba8b07bac90fc9c, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bbbc3c6ecb64467b9a5008d16d94ab2f, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/3904e5f3c4cc4720b4d4f4028b446cfd, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bc2e2df67cc64d7188cb56ce1d6632cf, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/786cfc6f95b24be29e1d8ea20a5fe085, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/34960061cb5a4cca94f5bdd537afdb57, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/d92f0acf43434b10bcb2ee30dc7c1578, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0608098ce44f4055b7697c2a991074a8, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/337a4821eedb4e6dbcf12853a63040a8, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ce0018d1299d474b97b20bac27ef097b, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0de993ca371b49ddbfdf36dc2fa07461] to archive 2024-12-15T06:18:16,211 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-15T06:18:16,214 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/1b65be837ec34957aa57cfd4be1a614c.c66dfc53ca05a54bcf1b32e5070c60b0 2024-12-15T06:18:16,214 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/97174d5c529343018f6af0cb054c1d0f to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/97174d5c529343018f6af0cb054c1d0f 2024-12-15T06:18:16,214 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-0e7809a140424cbe85a6a20525c10a35 2024-12-15T06:18:16,214 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/7175813ab1914bcf8cae42d33e41c288 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/7175813ab1914bcf8cae42d33e41c288 2024-12-15T06:18:16,214 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/TestLogRolling-testLogRolling=c66dfc53ca05a54bcf1b32e5070c60b0-7b3fc934f1fd422c8ed4fd72cc437135 2024-12-15T06:18:16,215 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e02a9de5d5e2489dba420443be289406 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/e02a9de5d5e2489dba420443be289406 2024-12-15T06:18:16,215 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ec74a92e80114002b5e3aa726f30e6ef to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ec74a92e80114002b5e3aa726f30e6ef 2024-12-15T06:18:16,215 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/c1c90f45dc1d4e718ba8b07bac90fc9c to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/c1c90f45dc1d4e718ba8b07bac90fc9c 2024-12-15T06:18:16,216 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/3904e5f3c4cc4720b4d4f4028b446cfd to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/3904e5f3c4cc4720b4d4f4028b446cfd 2024-12-15T06:18:16,217 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/d92f0acf43434b10bcb2ee30dc7c1578 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/d92f0acf43434b10bcb2ee30dc7c1578 2024-12-15T06:18:16,217 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bbbc3c6ecb64467b9a5008d16d94ab2f to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bbbc3c6ecb64467b9a5008d16d94ab2f 2024-12-15T06:18:16,217 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/786cfc6f95b24be29e1d8ea20a5fe085 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/786cfc6f95b24be29e1d8ea20a5fe085 2024-12-15T06:18:16,217 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/34960061cb5a4cca94f5bdd537afdb57 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/34960061cb5a4cca94f5bdd537afdb57 2024-12-15T06:18:16,217 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bc2e2df67cc64d7188cb56ce1d6632cf to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/bc2e2df67cc64d7188cb56ce1d6632cf 2024-12-15T06:18:16,218 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/337a4821eedb4e6dbcf12853a63040a8 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/337a4821eedb4e6dbcf12853a63040a8 2024-12-15T06:18:16,218 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0608098ce44f4055b7697c2a991074a8 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0608098ce44f4055b7697c2a991074a8 2024-12-15T06:18:16,218 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ce0018d1299d474b97b20bac27ef097b to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/ce0018d1299d474b97b20bac27ef097b 2024-12-15T06:18:16,219 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0de993ca371b49ddbfdf36dc2fa07461 to hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/archive/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/info/0de993ca371b49ddbfdf36dc2fa07461 2024-12-15T06:18:16,222 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/data/default/TestLogRolling-testLogRolling/da9f6afa8670a797d04086949b9e35c0/recovered.edits/328.seqid, newMaxSeqId=328, maxSeqId=122 2024-12-15T06:18:16,223 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:16,223 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for da9f6afa8670a797d04086949b9e35c0: 2024-12-15T06:18:16,223 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1734243443625.da9f6afa8670a797d04086949b9e35c0. 2024-12-15T06:18:16,399 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,37021,1734243418154; all regions closed. 2024-12-15T06:18:16,400 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154 2024-12-15T06:18:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741834_1010 (size=9351) 2024-12-15T06:18:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741834_1010 (size=9351) 2024-12-15T06:18:16,404 DEBUG [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/oldWALs 2024-12-15T06:18:16,404 INFO [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C37021%2C1734243418154.meta:.meta(num 1734243418887) 2024-12-15T06:18:16,404 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/WALs/00a5f8d66132,37021,1734243418154 2024-12-15T06:18:16,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741876_1052 (size=1071) 2024-12-15T06:18:16,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741876_1052 (size=1071) 2024-12-15T06:18:16,406 INFO [regionserver/00a5f8d66132:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-15T06:18:16,406 INFO [regionserver/00a5f8d66132:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-15T06:18:16,407 DEBUG [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/oldWALs 2024-12-15T06:18:16,407 INFO [RS:0;00a5f8d66132:37021 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C37021%2C1734243418154:(num 1734243496087) 2024-12-15T06:18:16,407 DEBUG [RS:0;00a5f8d66132:37021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:16,408 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:18:16,408 INFO [RS:0;00a5f8d66132:37021 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-15T06:18:16,408 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:18:16,408 INFO [RS:0;00a5f8d66132:37021 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:37021 2024-12-15T06:18:16,408 INFO [regionserver/00a5f8d66132:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:18:16,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,37021,1734243418154 2024-12-15T06:18:16,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:18:16,411 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,37021,1734243418154] 2024-12-15T06:18:16,411 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,37021,1734243418154; numProcessing=1 2024-12-15T06:18:16,412 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,37021,1734243418154 already deleted, retry=false 2024-12-15T06:18:16,412 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,37021,1734243418154 expired; onlineServers=0 2024-12-15T06:18:16,412 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,39109,1734243418107' ***** 2024-12-15T06:18:16,412 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-15T06:18:16,413 DEBUG [M:0;00a5f8d66132:39109 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcbb192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:18:16,413 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,39109,1734243418107 2024-12-15T06:18:16,413 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,39109,1734243418107; all regions closed. 2024-12-15T06:18:16,413 DEBUG [M:0;00a5f8d66132:39109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:16,413 DEBUG [M:0;00a5f8d66132:39109 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-15T06:18:16,413 DEBUG [M:0;00a5f8d66132:39109 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-15T06:18:16,413 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-15T06:18:16,413 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243418292 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243418292,5,FailOnTimeoutGroup] 2024-12-15T06:18:16,413 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243418292 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243418292,5,FailOnTimeoutGroup] 2024-12-15T06:18:16,413 INFO [M:0;00a5f8d66132:39109 {}] hbase.ChoreService(370): Chore service for: master/00a5f8d66132:0 had [] on shutdown 2024-12-15T06:18:16,413 DEBUG [M:0;00a5f8d66132:39109 {}] master.HMaster(1733): Stopping service threads 2024-12-15T06:18:16,413 INFO [M:0;00a5f8d66132:39109 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-15T06:18:16,413 INFO [M:0;00a5f8d66132:39109 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-15T06:18:16,414 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-15T06:18:16,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-15T06:18:16,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:16,414 DEBUG [M:0;00a5f8d66132:39109 {}] zookeeper.ZKUtil(347): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-15T06:18:16,414 WARN [M:0;00a5f8d66132:39109 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-15T06:18:16,414 INFO [M:0;00a5f8d66132:39109 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-15T06:18:16,415 INFO [M:0;00a5f8d66132:39109 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-15T06:18:16,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:18:16,415 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:18:16,415 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:16,415 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:16,415 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:18:16,415 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:16,415 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=66.42 KB heapSize=81.66 KB 2024-12-15T06:18:16,430 DEBUG [M:0;00a5f8d66132:39109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a7257d92d26c47d8bf23fe0d96fde642 is 82, key is hbase:meta,,1/info:regioninfo/1734243418906/Put/seqid=0 2024-12-15T06:18:16,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741877_1053 (size=5672) 2024-12-15T06:18:16,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741877_1053 (size=5672) 2024-12-15T06:18:16,435 INFO [M:0;00a5f8d66132:39109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a7257d92d26c47d8bf23fe0d96fde642 2024-12-15T06:18:16,454 DEBUG [M:0;00a5f8d66132:39109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7b8e4439335741769cd96fe42aa42666 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734243419751/Put/seqid=0 2024-12-15T06:18:16,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741878_1054 (size=7284) 2024-12-15T06:18:16,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741878_1054 (size=7284) 2024-12-15T06:18:16,459 INFO [M:0;00a5f8d66132:39109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65.82 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7b8e4439335741769cd96fe42aa42666 2024-12-15T06:18:16,462 INFO [M:0;00a5f8d66132:39109 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7b8e4439335741769cd96fe42aa42666 2024-12-15T06:18:16,476 DEBUG [M:0;00a5f8d66132:39109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/768508b00f874eb488fd78b552e2433e is 69, key is 00a5f8d66132,37021,1734243418154/rs:state/1734243418395/Put/seqid=0 2024-12-15T06:18:16,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741879_1055 (size=5156) 2024-12-15T06:18:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741879_1055 (size=5156) 2024-12-15T06:18:16,481 INFO [M:0;00a5f8d66132:39109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/768508b00f874eb488fd78b552e2433e 2024-12-15T06:18:16,499 DEBUG [M:0;00a5f8d66132:39109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9e609eac31e54c4ca1505c72bed5915f is 52, key is load_balancer_on/state:d/1734243419387/Put/seqid=0 2024-12-15T06:18:16,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741880_1056 (size=5056) 2024-12-15T06:18:16,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741880_1056 (size=5056) 2024-12-15T06:18:16,505 INFO [M:0;00a5f8d66132:39109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9e609eac31e54c4ca1505c72bed5915f 2024-12-15T06:18:16,510 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a7257d92d26c47d8bf23fe0d96fde642 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a7257d92d26c47d8bf23fe0d96fde642 2024-12-15T06:18:16,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:16,511 INFO [RS:0;00a5f8d66132:37021 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,37021,1734243418154; zookeeper connection closed. 2024-12-15T06:18:16,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37021-0x100945238d30001, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:16,512 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@646a1b9b {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@646a1b9b 2024-12-15T06:18:16,512 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-15T06:18:16,515 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a7257d92d26c47d8bf23fe0d96fde642, entries=8, sequenceid=164, filesize=5.5 K 2024-12-15T06:18:16,516 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7b8e4439335741769cd96fe42aa42666 as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7b8e4439335741769cd96fe42aa42666 2024-12-15T06:18:16,520 INFO [M:0;00a5f8d66132:39109 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7b8e4439335741769cd96fe42aa42666 2024-12-15T06:18:16,520 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7b8e4439335741769cd96fe42aa42666, entries=18, sequenceid=164, filesize=7.1 K 2024-12-15T06:18:16,520 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/768508b00f874eb488fd78b552e2433e as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/768508b00f874eb488fd78b552e2433e 2024-12-15T06:18:16,524 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/768508b00f874eb488fd78b552e2433e, entries=1, sequenceid=164, filesize=5.0 K 2024-12-15T06:18:16,525 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9e609eac31e54c4ca1505c72bed5915f as hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9e609eac31e54c4ca1505c72bed5915f 2024-12-15T06:18:16,528 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43087/user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9e609eac31e54c4ca1505c72bed5915f, entries=1, sequenceid=164, filesize=4.9 K 2024-12-15T06:18:16,529 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(3040): Finished flush of dataSize ~66.42 KB/68019, heapSize ~81.60 KB/83560, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=164, compaction requested=false 2024-12-15T06:18:16,530 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:16,530 DEBUG [M:0;00a5f8d66132:39109 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:18:16,531 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7de506d1-4e3b-dcac-f4e4-7196e25562a5/MasterData/WALs/00a5f8d66132,39109,1734243418107 2024-12-15T06:18:16,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42333 is added to blk_1073741830_1006 (size=79248) 2024-12-15T06:18:16,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34657 is added to blk_1073741830_1006 (size=79248) 2024-12-15T06:18:16,533 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:18:16,533 INFO [M:0;00a5f8d66132:39109 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-15T06:18:16,533 INFO [M:0;00a5f8d66132:39109 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:39109 2024-12-15T06:18:16,535 DEBUG [M:0;00a5f8d66132:39109 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/00a5f8d66132,39109,1734243418107 already deleted, retry=false 2024-12-15T06:18:16,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:16,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39109-0x100945238d30000, quorum=127.0.0.1:52277, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:16,636 INFO [M:0;00a5f8d66132:39109 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,39109,1734243418107; zookeeper connection closed. 2024-12-15T06:18:16,639 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68e91904{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:18:16,640 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3301755{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:18:16,640 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:18:16,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36c195b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:18:16,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d35a3c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir/,STOPPED} 2024-12-15T06:18:16,642 WARN [BP-906873066-172.17.0.2-1734243417453 heartbeating to localhost/127.0.0.1:43087 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:18:16,642 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:18:16,642 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:18:16,642 WARN [BP-906873066-172.17.0.2-1734243417453 heartbeating to localhost/127.0.0.1:43087 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-906873066-172.17.0.2-1734243417453 (Datanode Uuid 82144eaa-f56a-4e84-9e02-3126f4b854e8) service to localhost/127.0.0.1:43087 2024-12-15T06:18:16,642 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data3/current/BP-906873066-172.17.0.2-1734243417453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:16,643 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data4/current/BP-906873066-172.17.0.2-1734243417453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:16,643 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:18:16,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30633a18{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:18:16,645 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5334bcb3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:18:16,645 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:18:16,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26c4edf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:18:16,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d6992ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir/,STOPPED} 2024-12-15T06:18:16,646 WARN [BP-906873066-172.17.0.2-1734243417453 heartbeating to localhost/127.0.0.1:43087 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:18:16,646 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:18:16,646 WARN [BP-906873066-172.17.0.2-1734243417453 heartbeating to localhost/127.0.0.1:43087 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-906873066-172.17.0.2-1734243417453 (Datanode Uuid 0d054f23-d1b4-4182-8ac0-d2880884fe10) service to localhost/127.0.0.1:43087 2024-12-15T06:18:16,646 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:18:16,647 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data1/current/BP-906873066-172.17.0.2-1734243417453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:16,647 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/cluster_05519c52-55e9-ed0e-eb2a-a02369c7f81e/dfs/data/data2/current/BP-906873066-172.17.0.2-1734243417453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:16,648 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:18:16,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@342c9251{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:18:16,654 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@50679d28{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:18:16,654 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:18:16,654 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d22749a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:18:16,654 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@753c9dd9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir/,STOPPED} 2024-12-15T06:18:16,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-15T06:18:16,689 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-15T06:18:16,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=136 (was 115) - Thread LEAK? -, OpenFileDescriptor=493 (was 464) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=55 (was 38) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4157 (was 4195) 2024-12-15T06:18:16,703 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=136, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=55, ProcessCount=11, AvailableMemoryMB=4158 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.log.dir so I do NOT create it in target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d0185466-da10-beb1-7034-be2eff28b345/hadoop.tmp.dir so I do NOT create it in target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58, deleteOnExit=true 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/test.cache.data in system properties and HBase conf 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.tmp.dir in system properties and HBase conf 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir in system properties and HBase conf 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-15T06:18:16,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-15T06:18:16,704 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/nfs.dump.dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/java.io.tmpdir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-15T06:18:16,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-15T06:18:16,718 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:18:16,786 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:18:16,790 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:18:16,792 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:18:16,792 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:18:16,792 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:18:16,793 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:18:16,793 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e59f9a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:18:16,794 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77c62a33{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:18:16,898 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:16,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@542475d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/java.io.tmpdir/jetty-localhost-39231-hadoop-hdfs-3_4_1-tests_jar-_-any-3461574311938557398/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:18:16,924 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6055c262{HTTP/1.1, (http/1.1)}{localhost:39231} 2024-12-15T06:18:16,924 INFO [Time-limited test {}] server.Server(415): Started @374053ms 2024-12-15T06:18:16,937 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-15T06:18:16,988 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:18:16,991 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:18:16,992 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:18:16,992 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:18:16,992 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-15T06:18:16,993 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f60e8fe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:18:16,993 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@106e8f5b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:18:17,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6283594c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/java.io.tmpdir/jetty-localhost-42375-hadoop-hdfs-3_4_1-tests_jar-_-any-17501443447808594616/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:18:17,108 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@73e72bf4{HTTP/1.1, (http/1.1)}{localhost:42375} 2024-12-15T06:18:17,108 INFO [Time-limited test {}] server.Server(415): Started @374236ms 2024-12-15T06:18:17,109 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:18:17,136 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-15T06:18:17,138 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-15T06:18:17,139 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-15T06:18:17,139 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-15T06:18:17,139 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-15T06:18:17,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d8ddb07{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir/,AVAILABLE} 2024-12-15T06:18:17,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35cb60df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-15T06:18:17,186 WARN [Thread-2190 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data2/current/BP-1820409873-172.17.0.2-1734243496733/current, will proceed with Du for space computation calculation, 2024-12-15T06:18:17,186 WARN [Thread-2189 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data1/current/BP-1820409873-172.17.0.2-1734243496733/current, will proceed with Du for space computation calculation, 2024-12-15T06:18:17,202 WARN [Thread-2168 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:18:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8bfaa0c1bd817e8 with lease ID 0x9b7369f671080012: Processing first storage report for DS-7211f651-f79f-45c8-8716-32bbd5783d19 from datanode DatanodeRegistration(127.0.0.1:45053, datanodeUuid=30ed949f-b30b-49a8-acaa-ce974528142f, infoPort=38947, infoSecurePort=0, ipcPort=39375, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733) 2024-12-15T06:18:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8bfaa0c1bd817e8 with lease ID 0x9b7369f671080012: from storage DS-7211f651-f79f-45c8-8716-32bbd5783d19 node DatanodeRegistration(127.0.0.1:45053, datanodeUuid=30ed949f-b30b-49a8-acaa-ce974528142f, infoPort=38947, infoSecurePort=0, ipcPort=39375, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:18:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8bfaa0c1bd817e8 with lease ID 0x9b7369f671080012: Processing first storage report for DS-b6e66887-aa9f-4229-bea2-866938c8449d from datanode DatanodeRegistration(127.0.0.1:45053, datanodeUuid=30ed949f-b30b-49a8-acaa-ce974528142f, infoPort=38947, infoSecurePort=0, ipcPort=39375, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733) 2024-12-15T06:18:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8bfaa0c1bd817e8 with lease ID 0x9b7369f671080012: from storage DS-b6e66887-aa9f-4229-bea2-866938c8449d node DatanodeRegistration(127.0.0.1:45053, datanodeUuid=30ed949f-b30b-49a8-acaa-ce974528142f, infoPort=38947, infoSecurePort=0, ipcPort=39375, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:18:17,256 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6b08cc23{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/java.io.tmpdir/jetty-localhost-41281-hadoop-hdfs-3_4_1-tests_jar-_-any-9567069753316083510/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:18:17,257 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b3758d9{HTTP/1.1, (http/1.1)}{localhost:41281} 2024-12-15T06:18:17,257 INFO [Time-limited test {}] server.Server(415): Started @374386ms 2024-12-15T06:18:17,258 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-15T06:18:17,335 WARN [Thread-2215 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data3/current/BP-1820409873-172.17.0.2-1734243496733/current, will proceed with Du for space computation calculation, 2024-12-15T06:18:17,335 WARN [Thread-2216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data4/current/BP-1820409873-172.17.0.2-1734243496733/current, will proceed with Du for space computation calculation, 2024-12-15T06:18:17,352 WARN [Thread-2204 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-15T06:18:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedcbd2315d55e2e7 with lease ID 0x9b7369f671080013: Processing first storage report for DS-0b1f9b32-cccb-4489-86a4-f361a7e3bde4 from datanode DatanodeRegistration(127.0.0.1:40157, datanodeUuid=be92d954-0bc8-4711-9ccd-2bafb9781cde, infoPort=45515, infoSecurePort=0, ipcPort=36253, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733) 2024-12-15T06:18:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedcbd2315d55e2e7 with lease ID 0x9b7369f671080013: from storage DS-0b1f9b32-cccb-4489-86a4-f361a7e3bde4 node DatanodeRegistration(127.0.0.1:40157, datanodeUuid=be92d954-0bc8-4711-9ccd-2bafb9781cde, infoPort=45515, infoSecurePort=0, ipcPort=36253, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:18:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedcbd2315d55e2e7 with lease ID 0x9b7369f671080013: Processing first storage report for DS-8433285f-9ae9-4977-b34d-55a05c456ab3 from datanode DatanodeRegistration(127.0.0.1:40157, datanodeUuid=be92d954-0bc8-4711-9ccd-2bafb9781cde, infoPort=45515, infoSecurePort=0, ipcPort=36253, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733) 2024-12-15T06:18:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedcbd2315d55e2e7 with lease ID 0x9b7369f671080013: from storage DS-8433285f-9ae9-4977-b34d-55a05c456ab3 node DatanodeRegistration(127.0.0.1:40157, datanodeUuid=be92d954-0bc8-4711-9ccd-2bafb9781cde, infoPort=45515, infoSecurePort=0, ipcPort=36253, storageInfo=lv=-57;cid=testClusterID;nsid=1416490588;c=1734243496733), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-15T06:18:17,380 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00 2024-12-15T06:18:17,383 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/zookeeper_0, clientPort=60692, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-15T06:18:17,384 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60692 2024-12-15T06:18:17,384 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:18:17,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741825_1001 (size=7) 2024-12-15T06:18:17,394 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656 with version=8 2024-12-15T06:18:17,394 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:35345/user/jenkins/test-data/88753134-ba0a-3d25-4b4e-6e73b15e3659/hbase-staging 2024-12-15T06:18:17,396 INFO [Time-limited test {}] client.ConnectionUtils(129): master/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:18:17,396 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:18:17,397 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38539 2024-12-15T06:18:17,397 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,398 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:38539 connecting to ZooKeeper ensemble=127.0.0.1:60692 2024-12-15T06:18:17,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385390x0, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:18:17,406 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38539-0x10094536e8d0000 connected 2024-12-15T06:18:17,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:18:17,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:18:17,419 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:18:17,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38539 2024-12-15T06:18:17,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38539 2024-12-15T06:18:17,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38539 2024-12-15T06:18:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38539 2024-12-15T06:18:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38539 2024-12-15T06:18:17,426 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656, hbase.cluster.distributed=false 2024-12-15T06:18:17,442 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/00a5f8d66132:0 server-side Connection retries=45 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-15T06:18:17,442 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-15T06:18:17,443 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38237 2024-12-15T06:18:17,443 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-15T06:18:17,444 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-15T06:18:17,444 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,446 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,448 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:38237 connecting to ZooKeeper ensemble=127.0.0.1:60692 2024-12-15T06:18:17,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382370x0, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-15T06:18:17,451 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382370x0, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:18:17,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38237-0x10094536e8d0001 connected 2024-12-15T06:18:17,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:18:17,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-15T06:18:17,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38237 2024-12-15T06:18:17,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38237 2024-12-15T06:18:17,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38237 2024-12-15T06:18:17,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38237 2024-12-15T06:18:17,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38237 2024-12-15T06:18:17,458 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:18:17,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:18:17,460 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:18:17,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-15T06:18:17,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,462 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:18:17,463 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00a5f8d66132,38539,1734243497395 from backup master directory 2024-12-15T06:18:17,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:18:17,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-15T06:18:17,464 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:18:17,464 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,464 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-15T06:18:17,470 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00a5f8d66132:38539 2024-12-15T06:18:17,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:18:17,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741826_1002 (size=42) 2024-12-15T06:18:17,480 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/hbase.id with ID: ff5e88e7-2a88-4e7d-a20b-214e3ff3d8cd 2024-12-15T06:18:17,491 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:17,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:18:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741827_1003 (size=196) 2024-12-15T06:18:17,500 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-15T06:18:17,501 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-15T06:18:17,501 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:18:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:18:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741828_1004 (size=1189) 2024-12-15T06:18:17,508 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store 2024-12-15T06:18:17,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:18:17,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741829_1005 (size=34) 2024-12-15T06:18:17,514 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:18:17,515 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:18:17,515 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:17,515 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:17,515 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:18:17,515 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:17,515 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:17,515 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:18:17,515 WARN [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/.initializing 2024-12-15T06:18:17,516 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/WALs/00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,518 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C38539%2C1734243497395, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/WALs/00a5f8d66132,38539,1734243497395, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/oldWALs, maxLogs=10 2024-12-15T06:18:17,518 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C38539%2C1734243497395.1734243497518 2024-12-15T06:18:17,526 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/WALs/00a5f8d66132,38539,1734243497395/00a5f8d66132%2C38539%2C1734243497395.1734243497518 2024-12-15T06:18:17,526 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45515:45515),(127.0.0.1/127.0.0.1:38947:38947)] 2024-12-15T06:18:17,526 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:18:17,526 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:18:17,526 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,526 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-15T06:18:17,529 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:17,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-15T06:18:17,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:18:17,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-15T06:18:17,532 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:18:17,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-15T06:18:17,533 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:18:17,534 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,535 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,536 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-15T06:18:17,537 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-15T06:18:17,540 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:18:17,540 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754682, jitterRate=-0.040373072028160095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-15T06:18:17,541 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:18:17,541 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-15T06:18:17,544 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f51e1b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:18:17,544 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-15T06:18:17,545 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-15T06:18:17,545 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-15T06:18:17,545 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-15T06:18:17,545 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-15T06:18:17,545 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-15T06:18:17,545 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-15T06:18:17,547 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-15T06:18:17,548 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-15T06:18:17,549 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-15T06:18:17,550 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-15T06:18:17,550 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-15T06:18:17,551 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-15T06:18:17,551 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-15T06:18:17,552 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-15T06:18:17,553 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-15T06:18:17,553 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-15T06:18:17,555 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-15T06:18:17,556 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-15T06:18:17,557 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-15T06:18:17,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:18:17,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-15T06:18:17,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,559 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=00a5f8d66132,38539,1734243497395, sessionid=0x10094536e8d0000, setting cluster-up flag (Was=false) 2024-12-15T06:18:17,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,565 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-15T06:18:17,566 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:17,572 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-15T06:18:17,573 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00a5f8d66132,38539,1734243497395 2024-12-15T06:18:17,576 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-15T06:18:17,576 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-15T06:18:17,576 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00a5f8d66132,38539,1734243497395 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00a5f8d66132:0, corePoolSize=5, maxPoolSize=5 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00a5f8d66132:0, corePoolSize=10, maxPoolSize=10 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:18:17,577 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,578 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:18:17,578 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-15T06:18:17,579 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,579 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:18:17,582 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734243527582 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-15T06:18:17,583 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-15T06:18:17,584 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-15T06:18:17,586 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-15T06:18:17,586 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-15T06:18:17,588 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243497586,5,FailOnTimeoutGroup] 2024-12-15T06:18:17,592 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243497588,5,FailOnTimeoutGroup] 2024-12-15T06:18:17,592 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,592 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-15T06:18:17,592 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,592 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:18:17,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741831_1007 (size=1039) 2024-12-15T06:18:17,595 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-15T06:18:17,595 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656 2024-12-15T06:18:17,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:18:17,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741832_1008 (size=32) 2024-12-15T06:18:17,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:18:17,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:18:17,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:18:17,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:17,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:18:17,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:18:17,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:17,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:18:17,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:18:17,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:17,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:17,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740 2024-12-15T06:18:17,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740 2024-12-15T06:18:17,615 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:18:17,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:18:17,618 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702660, jitterRate=-0.10652250051498413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:18:17,618 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:18:17,618 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:18:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:18:17,619 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-15T06:18:17,619 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-15T06:18:17,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-15T06:18:17,620 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-15T06:18:17,621 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-15T06:18:17,670 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00a5f8d66132:38237 2024-12-15T06:18:17,671 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1008): ClusterId : ff5e88e7-2a88-4e7d-a20b-214e3ff3d8cd 2024-12-15T06:18:17,671 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-15T06:18:17,673 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-15T06:18:17,673 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-15T06:18:17,675 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-15T06:18:17,676 DEBUG [RS:0;00a5f8d66132:38237 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9506a13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:18:17,676 DEBUG [RS:0;00a5f8d66132:38237 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4912cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:18:17,676 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-15T06:18:17,676 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-15T06:18:17,676 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-15T06:18:17,677 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(3073): reportForDuty to master=00a5f8d66132,38539,1734243497395 with isa=00a5f8d66132/172.17.0.2:38237, startcode=1734243497442 2024-12-15T06:18:17,677 DEBUG [RS:0;00a5f8d66132:38237 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-15T06:18:17,679 INFO [RS-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57995, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-15T06:18:17,680 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38539 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 00a5f8d66132,38237,1734243497442 2024-12-15T06:18:17,680 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38539 {}] master.ServerManager(486): Registering regionserver=00a5f8d66132,38237,1734243497442 2024-12-15T06:18:17,681 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656 2024-12-15T06:18:17,681 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35361 2024-12-15T06:18:17,681 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-15T06:18:17,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:18:17,683 DEBUG [RS:0;00a5f8d66132:38237 {}] zookeeper.ZKUtil(111): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00a5f8d66132,38237,1734243497442 2024-12-15T06:18:17,683 WARN [RS:0;00a5f8d66132:38237 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-15T06:18:17,683 INFO [RS:0;00a5f8d66132:38237 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:18:17,683 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442 2024-12-15T06:18:17,683 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00a5f8d66132,38237,1734243497442] 2024-12-15T06:18:17,686 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-15T06:18:17,687 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-15T06:18:17,688 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-15T06:18:17,689 INFO [RS:0;00a5f8d66132:38237 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-15T06:18:17,689 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,689 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-15T06:18:17,690 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00a5f8d66132:0, corePoolSize=2, maxPoolSize=2 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00a5f8d66132:0, corePoolSize=1, maxPoolSize=1 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:18:17,690 DEBUG [RS:0;00a5f8d66132:38237 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00a5f8d66132:0, corePoolSize=3, maxPoolSize=3 2024-12-15T06:18:17,692 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,692 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,692 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,692 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,692 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38237,1734243497442-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:18:17,706 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-15T06:18:17,706 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38237,1734243497442-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:17,721 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.Replication(204): 00a5f8d66132,38237,1734243497442 started 2024-12-15T06:18:17,721 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1767): Serving as 00a5f8d66132,38237,1734243497442, RpcServer on 00a5f8d66132/172.17.0.2:38237, sessionid=0x10094536e8d0001 2024-12-15T06:18:17,721 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-15T06:18:17,721 DEBUG [RS:0;00a5f8d66132:38237 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00a5f8d66132,38237,1734243497442 2024-12-15T06:18:17,721 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,38237,1734243497442' 2024-12-15T06:18:17,721 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-15T06:18:17,721 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00a5f8d66132,38237,1734243497442 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00a5f8d66132,38237,1734243497442' 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-15T06:18:17,722 DEBUG [RS:0;00a5f8d66132:38237 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-15T06:18:17,722 INFO [RS:0;00a5f8d66132:38237 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-15T06:18:17,722 INFO [RS:0;00a5f8d66132:38237 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-15T06:18:17,771 WARN [00a5f8d66132:38539 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-15T06:18:17,824 INFO [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C38237%2C1734243497442, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs, maxLogs=32 2024-12-15T06:18:17,825 INFO [RS:0;00a5f8d66132:38237 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C38237%2C1734243497442.1734243497825 2024-12-15T06:18:17,831 INFO [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442/00a5f8d66132%2C38237%2C1734243497442.1734243497825 2024-12-15T06:18:17,831 DEBUG [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38947:38947),(127.0.0.1/127.0.0.1:45515:45515)] 2024-12-15T06:18:17,899 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:18,021 DEBUG [00a5f8d66132:38539 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-15T06:18:18,022 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,023 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,38237,1734243497442, state=OPENING 2024-12-15T06:18:18,024 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-15T06:18:18,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:18,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:18,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=00a5f8d66132,38237,1734243497442}] 2024-12-15T06:18:18,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:18:18,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:18:18,178 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,178 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-15T06:18:18,180 INFO [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60722, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-15T06:18:18,184 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-15T06:18:18,184 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:18:18,187 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00a5f8d66132%2C38237%2C1734243497442.meta, suffix=.meta, logDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs, maxLogs=32 2024-12-15T06:18:18,187 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00a5f8d66132%2C38237%2C1734243497442.meta.1734243498187.meta 2024-12-15T06:18:18,194 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442/00a5f8d66132%2C38237%2C1734243497442.meta.1734243498187.meta 2024-12-15T06:18:18,194 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45515:45515),(127.0.0.1/127.0.0.1:38947:38947)] 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-15T06:18:18,195 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-15T06:18:18,195 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-15T06:18:18,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-15T06:18:18,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-15T06:18:18,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:18,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:18,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-15T06:18:18,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-15T06:18:18,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:18,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:18,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-15T06:18:18,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-15T06:18:18,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:18,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-15T06:18:18,200 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740 2024-12-15T06:18:18,201 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740 2024-12-15T06:18:18,202 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-15T06:18:18,204 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-15T06:18:18,205 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836171, jitterRate=0.0632474273443222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-15T06:18:18,205 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-15T06:18:18,205 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734243498178 2024-12-15T06:18:18,207 DEBUG [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-15T06:18:18,207 INFO [RS_OPEN_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-15T06:18:18,208 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,208 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00a5f8d66132,38237,1734243497442, state=OPEN 2024-12-15T06:18:18,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:18:18,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-15T06:18:18,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:18:18,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-15T06:18:18,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-15T06:18:18,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=00a5f8d66132,38237,1734243497442 in 186 msec 2024-12-15T06:18:18,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-15T06:18:18,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 595 msec 2024-12-15T06:18:18,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 640 msec 2024-12-15T06:18:18,217 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734243498217, completionTime=-1 2024-12-15T06:18:18,217 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-15T06:18:18,217 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-15T06:18:18,217 DEBUG [hconnection-0x8b64a27-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:18:18,219 INFO [RS-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60724, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:18:18,219 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-15T06:18:18,220 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734243558219 2024-12-15T06:18:18,220 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734243618220 2024-12-15T06:18:18,220 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38539,1734243497395-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38539,1734243497395-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38539,1734243497395-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00a5f8d66132:38539, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-15T06:18:18,225 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-15T06:18:18,226 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-15T06:18:18,226 DEBUG [master/00a5f8d66132:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-15T06:18:18,227 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-15T06:18:18,227 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:18,228 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-15T06:18:18,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:18:18,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741835_1011 (size=358) 2024-12-15T06:18:18,236 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 69caa2d7f6e6501a715f75838f7ba9d5, NAME => 'hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656 2024-12-15T06:18:18,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:18:18,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741836_1012 (size=42) 2024-12-15T06:18:18,241 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:18:18,242 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 69caa2d7f6e6501a715f75838f7ba9d5, disabling compactions & flushes 2024-12-15T06:18:18,242 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,242 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,242 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. after waiting 0 ms 2024-12-15T06:18:18,242 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,242 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,242 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 69caa2d7f6e6501a715f75838f7ba9d5: 2024-12-15T06:18:18,243 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-15T06:18:18,243 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734243498243"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734243498243"}]},"ts":"1734243498243"} 2024-12-15T06:18:18,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-15T06:18:18,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-15T06:18:18,245 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243498245"}]},"ts":"1734243498245"} 2024-12-15T06:18:18,247 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-15T06:18:18,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=69caa2d7f6e6501a715f75838f7ba9d5, ASSIGN}] 2024-12-15T06:18:18,251 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=69caa2d7f6e6501a715f75838f7ba9d5, ASSIGN 2024-12-15T06:18:18,251 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=69caa2d7f6e6501a715f75838f7ba9d5, ASSIGN; state=OFFLINE, location=00a5f8d66132,38237,1734243497442; forceNewPlan=false, retain=false 2024-12-15T06:18:18,402 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=69caa2d7f6e6501a715f75838f7ba9d5, regionState=OPENING, regionLocation=00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 69caa2d7f6e6501a715f75838f7ba9d5, server=00a5f8d66132,38237,1734243497442}] 2024-12-15T06:18:18,556 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,559 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 69caa2d7f6e6501a715f75838f7ba9d5, NAME => 'hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5.', STARTKEY => '', ENDKEY => ''} 2024-12-15T06:18:18,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-15T06:18:18,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,561 INFO [StoreOpener-69caa2d7f6e6501a715f75838f7ba9d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,562 INFO [StoreOpener-69caa2d7f6e6501a715f75838f7ba9d5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69caa2d7f6e6501a715f75838f7ba9d5 columnFamilyName info 2024-12-15T06:18:18,562 DEBUG [StoreOpener-69caa2d7f6e6501a715f75838f7ba9d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-15T06:18:18,562 INFO [StoreOpener-69caa2d7f6e6501a715f75838f7ba9d5-1 {}] regionserver.HStore(327): Store=69caa2d7f6e6501a715f75838f7ba9d5/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-15T06:18:18,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-15T06:18:18,567 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 69caa2d7f6e6501a715f75838f7ba9d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=697643, jitterRate=-0.11290189623832703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-15T06:18:18,568 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 69caa2d7f6e6501a715f75838f7ba9d5: 2024-12-15T06:18:18,568 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5., pid=6, masterSystemTime=1734243498556 2024-12-15T06:18:18,570 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,570 INFO [RS_OPEN_PRIORITY_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,570 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=69caa2d7f6e6501a715f75838f7ba9d5, regionState=OPEN, openSeqNum=2, regionLocation=00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-15T06:18:18,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 69caa2d7f6e6501a715f75838f7ba9d5, server=00a5f8d66132,38237,1734243497442 in 169 msec 2024-12-15T06:18:18,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-15T06:18:18,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=69caa2d7f6e6501a715f75838f7ba9d5, ASSIGN in 323 msec 2024-12-15T06:18:18,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-15T06:18:18,576 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734243498576"}]},"ts":"1734243498576"} 2024-12-15T06:18:18,577 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-15T06:18:18,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-15T06:18:18,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 354 msec 2024-12-15T06:18:18,627 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-15T06:18:18,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:18,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:18:18,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:18,633 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-15T06:18:18,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:18:18,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 9 msec 2024-12-15T06:18:18,644 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-15T06:18:18,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-15T06:18:18,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 8 msec 2024-12-15T06:18:18,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-15T06:18:18,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-15T06:18:18,660 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.196sec 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38539,1734243497395-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-15T06:18:18,661 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38539,1734243497395-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-15T06:18:18,662 DEBUG [master/00a5f8d66132:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-15T06:18:18,662 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-15T06:18:18,663 INFO [master/00a5f8d66132:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00a5f8d66132,38539,1734243497395-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-15T06:18:18,760 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c169c77 to 127.0.0.1:60692 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d2dea33 2024-12-15T06:18:18,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9b0a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-15T06:18:18,765 DEBUG [hconnection-0x350a9650-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-15T06:18:18,767 INFO [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60728, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-15T06:18:18,768 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=00a5f8d66132,38539,1734243497395 2024-12-15T06:18:18,769 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-15T06:18:18,771 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-15T06:18:18,771 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-15T06:18:18,773 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs, maxLogs=32 2024-12-15T06:18:18,773 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1734243498773 2024-12-15T06:18:18,778 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1/test.com%2C8080%2C1.1734243498773 2024-12-15T06:18:18,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38947:38947),(127.0.0.1/127.0.0.1:45515:45515)] 2024-12-15T06:18:18,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1734243498778 2024-12-15T06:18:18,784 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1/test.com%2C8080%2C1.1734243498773 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1/test.com%2C8080%2C1.1734243498778 2024-12-15T06:18:18,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38947:38947),(127.0.0.1/127.0.0.1:45515:45515)] 2024-12-15T06:18:18,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1/test.com%2C8080%2C1.1734243498773 is not closed yet, will try archiving it next time 2024-12-15T06:18:18,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741837_1013 (size=93) 2024-12-15T06:18:18,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741837_1013 (size=93) 2024-12-15T06:18:18,790 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1 2024-12-15T06:18:18,790 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/test.com,8080,1/test.com%2C8080%2C1.1734243498773 to hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs/test.com%2C8080%2C1.1734243498773 2024-12-15T06:18:18,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741838_1014 (size=93) 2024-12-15T06:18:18,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741838_1014 (size=93) 2024-12-15T06:18:18,794 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs 2024-12-15T06:18:18,794 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1734243498778) 2024-12-15T06:18:18,794 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-15T06:18:18,794 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c169c77 to 127.0.0.1:60692 2024-12-15T06:18:18,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:18,794 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-15T06:18:18,794 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1804715911, stopped=false 2024-12-15T06:18:18,794 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=00a5f8d66132,38539,1734243497395 2024-12-15T06:18:18,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:18:18,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-15T06:18:18,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:18,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-15T06:18:18,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:18,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:18:18,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-15T06:18:18,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:18,801 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,38237,1734243497442' ***** 2024-12-15T06:18:18,801 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-15T06:18:18,802 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(3579): Received CLOSE for 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,38237,1734243497442 2024-12-15T06:18:18,802 DEBUG [RS:0;00a5f8d66132:38237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-15T06:18:18,802 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 69caa2d7f6e6501a715f75838f7ba9d5, disabling compactions & flushes 2024-12-15T06:18:18,802 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-15T06:18:18,802 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,802 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. after waiting 0 ms 2024-12-15T06:18:18,802 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:18,802 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 69caa2d7f6e6501a715f75838f7ba9d5 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-15T06:18:18,802 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-15T06:18:18,802 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 69caa2d7f6e6501a715f75838f7ba9d5=hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5.} 2024-12-15T06:18:18,803 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:18,803 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-15T06:18:18,803 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-15T06:18:18,803 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-15T06:18:18,803 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-15T06:18:18,803 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-15T06:18:18,803 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-12-15T06:18:18,817 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/.tmp/info/c0be54bfb8674cc796172691058d5c0d is 45, key is default/info:d/1734243498636/Put/seqid=0 2024-12-15T06:18:18,818 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/.tmp/info/e48b7d9d7c4a4a908aac9920d2550f86 is 143, key is hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5./info:regioninfo/1734243498570/Put/seqid=0 2024-12-15T06:18:18,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741840_1016 (size=6595) 2024-12-15T06:18:18,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741839_1015 (size=5037) 2024-12-15T06:18:18,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741839_1015 (size=5037) 2024-12-15T06:18:18,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741840_1016 (size=6595) 2024-12-15T06:18:18,899 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35953/user/jenkins/test-data/11050e94-06ca-502a-518c-1cf5aeaf774b/WALs/00a5f8d66132,37129,1734243249809/00a5f8d66132%2C37129%2C1734243249809.meta.1734243250640.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-15T06:18:19,003 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:19,203 DEBUG [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 69caa2d7f6e6501a715f75838f7ba9d5 2024-12-15T06:18:19,225 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/.tmp/info/c0be54bfb8674cc796172691058d5c0d 2024-12-15T06:18:19,225 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/.tmp/info/e48b7d9d7c4a4a908aac9920d2550f86 2024-12-15T06:18:19,231 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/.tmp/info/c0be54bfb8674cc796172691058d5c0d as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/info/c0be54bfb8674cc796172691058d5c0d 2024-12-15T06:18:19,235 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/info/c0be54bfb8674cc796172691058d5c0d, entries=2, sequenceid=6, filesize=4.9 K 2024-12-15T06:18:19,236 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 69caa2d7f6e6501a715f75838f7ba9d5 in 434ms, sequenceid=6, compaction requested=false 2024-12-15T06:18:19,236 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-15T06:18:19,240 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/namespace/69caa2d7f6e6501a715f75838f7ba9d5/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-15T06:18:19,240 INFO [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:19,240 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 69caa2d7f6e6501a715f75838f7ba9d5: 2024-12-15T06:18:19,240 DEBUG [RS_CLOSE_REGION-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734243498225.69caa2d7f6e6501a715f75838f7ba9d5. 2024-12-15T06:18:19,251 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/.tmp/table/2a4af9eb7416489b84178febabdc31fd is 51, key is hbase:namespace/table:state/1734243498576/Put/seqid=0 2024-12-15T06:18:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741841_1017 (size=5242) 2024-12-15T06:18:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741841_1017 (size=5242) 2024-12-15T06:18:19,256 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/.tmp/table/2a4af9eb7416489b84178febabdc31fd 2024-12-15T06:18:19,262 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/.tmp/info/e48b7d9d7c4a4a908aac9920d2550f86 as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/info/e48b7d9d7c4a4a908aac9920d2550f86 2024-12-15T06:18:19,266 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/info/e48b7d9d7c4a4a908aac9920d2550f86, entries=10, sequenceid=9, filesize=6.4 K 2024-12-15T06:18:19,267 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/.tmp/table/2a4af9eb7416489b84178febabdc31fd as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/table/2a4af9eb7416489b84178febabdc31fd 2024-12-15T06:18:19,271 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/table/2a4af9eb7416489b84178febabdc31fd, entries=2, sequenceid=9, filesize=5.1 K 2024-12-15T06:18:19,272 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 469ms, sequenceid=9, compaction requested=false 2024-12-15T06:18:19,272 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-15T06:18:19,276 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-15T06:18:19,277 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-15T06:18:19,277 INFO [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-15T06:18:19,277 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-15T06:18:19,277 DEBUG [RS_CLOSE_META-regionserver/00a5f8d66132:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-15T06:18:19,403 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,38237,1734243497442; all regions closed. 2024-12-15T06:18:19,404 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442 2024-12-15T06:18:19,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741834_1010 (size=2484) 2024-12-15T06:18:19,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741834_1010 (size=2484) 2024-12-15T06:18:19,407 DEBUG [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs 2024-12-15T06:18:19,407 INFO [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C38237%2C1734243497442.meta:.meta(num 1734243498187) 2024-12-15T06:18:19,408 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/WALs/00a5f8d66132,38237,1734243497442 2024-12-15T06:18:19,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741833_1009 (size=1414) 2024-12-15T06:18:19,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741833_1009 (size=1414) 2024-12-15T06:18:19,411 DEBUG [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/oldWALs 2024-12-15T06:18:19,411 INFO [RS:0;00a5f8d66132:38237 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 00a5f8d66132%2C38237%2C1734243497442:(num 1734243497825) 2024-12-15T06:18:19,411 DEBUG [RS:0;00a5f8d66132:38237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:19,411 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.LeaseManager(133): Closed leases 2024-12-15T06:18:19,411 INFO [RS:0;00a5f8d66132:38237 {}] hbase.ChoreService(370): Chore service for: regionserver/00a5f8d66132:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-15T06:18:19,411 INFO [regionserver/00a5f8d66132:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:18:19,411 INFO [RS:0;00a5f8d66132:38237 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38237 2024-12-15T06:18:19,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-15T06:18:19,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00a5f8d66132,38237,1734243497442 2024-12-15T06:18:19,414 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00a5f8d66132,38237,1734243497442] 2024-12-15T06:18:19,414 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 00a5f8d66132,38237,1734243497442; numProcessing=1 2024-12-15T06:18:19,416 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/00a5f8d66132,38237,1734243497442 already deleted, retry=false 2024-12-15T06:18:19,416 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 00a5f8d66132,38237,1734243497442 expired; onlineServers=0 2024-12-15T06:18:19,416 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '00a5f8d66132,38539,1734243497395' ***** 2024-12-15T06:18:19,416 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-15T06:18:19,416 DEBUG [M:0;00a5f8d66132:38539 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab8bdea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00a5f8d66132/172.17.0.2:0 2024-12-15T06:18:19,416 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegionServer(1224): stopping server 00a5f8d66132,38539,1734243497395 2024-12-15T06:18:19,416 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegionServer(1250): stopping server 00a5f8d66132,38539,1734243497395; all regions closed. 2024-12-15T06:18:19,416 DEBUG [M:0;00a5f8d66132:38539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-15T06:18:19,416 DEBUG [M:0;00a5f8d66132:38539 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-15T06:18:19,416 DEBUG [M:0;00a5f8d66132:38539 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-15T06:18:19,416 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-15T06:18:19,416 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243497586 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.large.0-1734243497586,5,FailOnTimeoutGroup] 2024-12-15T06:18:19,416 DEBUG [master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243497588 {}] cleaner.HFileCleaner(306): Exit Thread[master/00a5f8d66132:0:becomeActiveMaster-HFileCleaner.small.0-1734243497588,5,FailOnTimeoutGroup] 2024-12-15T06:18:19,416 INFO [M:0;00a5f8d66132:38539 {}] hbase.ChoreService(370): Chore service for: master/00a5f8d66132:0 had [] on shutdown 2024-12-15T06:18:19,417 DEBUG [M:0;00a5f8d66132:38539 {}] master.HMaster(1733): Stopping service threads 2024-12-15T06:18:19,417 INFO [M:0;00a5f8d66132:38539 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-15T06:18:19,417 INFO [M:0;00a5f8d66132:38539 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-15T06:18:19,417 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-15T06:18:19,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-15T06:18:19,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-15T06:18:19,418 DEBUG [M:0;00a5f8d66132:38539 {}] zookeeper.ZKUtil(347): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-15T06:18:19,418 WARN [M:0;00a5f8d66132:38539 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-15T06:18:19,418 INFO [M:0;00a5f8d66132:38539 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-15T06:18:19,418 INFO [M:0;00a5f8d66132:38539 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-15T06:18:19,418 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-15T06:18:19,418 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-15T06:18:19,418 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:19,418 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:19,418 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-15T06:18:19,418 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:19,418 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.32 KB heapSize=32.31 KB 2024-12-15T06:18:19,433 DEBUG [M:0;00a5f8d66132:38539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f38f25f9ba2d4d709bf6234570dcb59f is 82, key is hbase:meta,,1/info:regioninfo/1734243498207/Put/seqid=0 2024-12-15T06:18:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741842_1018 (size=5672) 2024-12-15T06:18:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741842_1018 (size=5672) 2024-12-15T06:18:19,438 INFO [M:0;00a5f8d66132:38539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f38f25f9ba2d4d709bf6234570dcb59f 2024-12-15T06:18:19,463 DEBUG [M:0;00a5f8d66132:38539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f5e1bae68c394b0e99b332eb4df90e4c is 696, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1734243498580/Put/seqid=0 2024-12-15T06:18:19,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741843_1019 (size=6626) 2024-12-15T06:18:19,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741843_1019 (size=6626) 2024-12-15T06:18:19,468 INFO [M:0;00a5f8d66132:38539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.72 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f5e1bae68c394b0e99b332eb4df90e4c 2024-12-15T06:18:19,486 DEBUG [M:0;00a5f8d66132:38539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/999720838b5749e29579868dc1fde1d0 is 69, key is 00a5f8d66132,38237,1734243497442/rs:state/1734243497680/Put/seqid=0 2024-12-15T06:18:19,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741844_1020 (size=5156) 2024-12-15T06:18:19,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741844_1020 (size=5156) 2024-12-15T06:18:19,490 INFO [M:0;00a5f8d66132:38539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/999720838b5749e29579868dc1fde1d0 2024-12-15T06:18:19,509 DEBUG [M:0;00a5f8d66132:38539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/211e23c548bd418ebd4b806f8cc6dec4 is 52, key is load_balancer_on/state:d/1734243498770/Put/seqid=0 2024-12-15T06:18:19,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741845_1021 (size=5056) 2024-12-15T06:18:19,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741845_1021 (size=5056) 2024-12-15T06:18:19,513 INFO [M:0;00a5f8d66132:38539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/211e23c548bd418ebd4b806f8cc6dec4 2024-12-15T06:18:19,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:19,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38237-0x10094536e8d0001, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:19,515 INFO [RS:0;00a5f8d66132:38237 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,38237,1734243497442; zookeeper connection closed. 2024-12-15T06:18:19,515 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2f52ed86 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2f52ed86 2024-12-15T06:18:19,515 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-15T06:18:19,518 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f38f25f9ba2d4d709bf6234570dcb59f as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f38f25f9ba2d4d709bf6234570dcb59f 2024-12-15T06:18:19,522 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f38f25f9ba2d4d709bf6234570dcb59f, entries=8, sequenceid=70, filesize=5.5 K 2024-12-15T06:18:19,522 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f5e1bae68c394b0e99b332eb4df90e4c as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f5e1bae68c394b0e99b332eb4df90e4c 2024-12-15T06:18:19,526 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f5e1bae68c394b0e99b332eb4df90e4c, entries=8, sequenceid=70, filesize=6.5 K 2024-12-15T06:18:19,527 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/999720838b5749e29579868dc1fde1d0 as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/999720838b5749e29579868dc1fde1d0 2024-12-15T06:18:19,530 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/999720838b5749e29579868dc1fde1d0, entries=1, sequenceid=70, filesize=5.0 K 2024-12-15T06:18:19,531 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/211e23c548bd418ebd4b806f8cc6dec4 as hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/211e23c548bd418ebd4b806f8cc6dec4 2024-12-15T06:18:19,535 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35361/user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/211e23c548bd418ebd4b806f8cc6dec4, entries=1, sequenceid=70, filesize=4.9 K 2024-12-15T06:18:19,536 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.32 KB/25929, heapSize ~32.25 KB/33024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=70, compaction requested=false 2024-12-15T06:18:19,537 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-15T06:18:19,537 DEBUG [M:0;00a5f8d66132:38539 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-15T06:18:19,538 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/97546425-98a7-b5bd-c8cc-96a4f18de656/MasterData/WALs/00a5f8d66132,38539,1734243497395 2024-12-15T06:18:19,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45053 is added to blk_1073741830_1006 (size=31030) 2024-12-15T06:18:19,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741830_1006 (size=31030) 2024-12-15T06:18:19,540 INFO [M:0;00a5f8d66132:38539 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-15T06:18:19,540 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-15T06:18:19,540 INFO [M:0;00a5f8d66132:38539 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38539 2024-12-15T06:18:19,542 DEBUG [M:0;00a5f8d66132:38539 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/00a5f8d66132,38539,1734243497395 already deleted, retry=false 2024-12-15T06:18:19,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:19,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38539-0x10094536e8d0000, quorum=127.0.0.1:60692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-15T06:18:19,644 INFO [M:0;00a5f8d66132:38539 {}] regionserver.HRegionServer(1307): Exiting; stopping=00a5f8d66132,38539,1734243497395; zookeeper connection closed. 2024-12-15T06:18:19,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6b08cc23{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:18:19,646 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b3758d9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:18:19,646 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:18:19,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35cb60df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:18:19,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d8ddb07{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir/,STOPPED} 2024-12-15T06:18:19,648 WARN [BP-1820409873-172.17.0.2-1734243496733 heartbeating to localhost/127.0.0.1:35361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:18:19,648 WARN [BP-1820409873-172.17.0.2-1734243496733 heartbeating to localhost/127.0.0.1:35361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1820409873-172.17.0.2-1734243496733 (Datanode Uuid be92d954-0bc8-4711-9ccd-2bafb9781cde) service to localhost/127.0.0.1:35361 2024-12-15T06:18:19,648 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:18:19,648 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:18:19,648 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data3/current/BP-1820409873-172.17.0.2-1734243496733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:19,649 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data4/current/BP-1820409873-172.17.0.2-1734243496733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:19,649 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:18:19,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6283594c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-15T06:18:19,651 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@73e72bf4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:18:19,651 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:18:19,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@106e8f5b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:18:19,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f60e8fe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir/,STOPPED} 2024-12-15T06:18:19,652 WARN [BP-1820409873-172.17.0.2-1734243496733 heartbeating to localhost/127.0.0.1:35361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-15T06:18:19,652 WARN [BP-1820409873-172.17.0.2-1734243496733 heartbeating to localhost/127.0.0.1:35361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1820409873-172.17.0.2-1734243496733 (Datanode Uuid 30ed949f-b30b-49a8-acaa-ce974528142f) service to localhost/127.0.0.1:35361 2024-12-15T06:18:19,652 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-15T06:18:19,652 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-15T06:18:19,653 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data1/current/BP-1820409873-172.17.0.2-1734243496733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:19,653 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/cluster_c916ed6d-a5ac-c379-9cb7-a0b63f597f58/dfs/data/data2/current/BP-1820409873-172.17.0.2-1734243496733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-15T06:18:19,653 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-15T06:18:19,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@542475d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-15T06:18:19,659 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6055c262{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-15T06:18:19,659 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-15T06:18:19,659 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77c62a33{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-15T06:18:19,659 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e59f9a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a20ece19-906b-1301-3663-eef3e17a9d00/hadoop.log.dir/,STOPPED} 2024-12-15T06:18:19,665 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-15T06:18:19,679 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-15T06:18:19,687 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=157 (was 136) - Thread LEAK? -, OpenFileDescriptor=519 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=55 (was 55), ProcessCount=11 (was 11), AvailableMemoryMB=4150 (was 4158)